01HPN9TQWCCP6J87DYA8KMPMN5: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HPN9V6HT5E9CEGDSMSNBX0M8

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-15T02:51:55.798Zjob dependencies complete; ready to run (waiting for 17 m 23 s)
22024-02-15T02:54:55.536Zjob assigned to worker 01HPNAYWK8S2S353JCFZJ8MPWW (queued for 2 m 59 s)
32024-02-15T02:54:56.027Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-15T02:55:22.587Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-15T02:55:22.587Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-15T02:55:47.053Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-15T02:55:47.053Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-15T02:56:04.626Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-15T02:56:04.626Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-15T02:56:27.775Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-15T02:56:27.780Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-15T02:56:27.793Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-15T02:56:27.793Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-15T02:56:27.829Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-15T02:56:27.829Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-15T02:56:27.848Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-15T02:56:27.848Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-15T02:56:27.862Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-15T02:56:27.862Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-15T02:56:27.880Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-15T02:56:27.881Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-15T02:56:27.903Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-15T02:56:27.903Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-15T02:56:27.939Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-15T02:56:27.949Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-15T02:56:44.896Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-15T02:56:44.896Zstarting task 0: "setup"
282024-02-15T02:56:45.145Z++ uname -s
292024-02-15T02:56:45.279Z+ kern=SunOS
302024-02-15T02:56:45.279Z+ case "$kern" in
312024-02-15T02:56:45.280Z+ groupadd -g 12345 build
322024-02-15T02:56:45.280Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-15T02:56:47.164Z+ zfs create -o mountpoint=/work rpool/work
342024-02-15T02:56:47.256Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-15T02:56:47.261Z+ home_fs=zfs
362024-02-15T02:56:47.261Z+ [[ zfs == autofs ]]
372024-02-15T02:56:47.261Z+ mkdir -p /home/build
382024-02-15T02:56:47.262Z+ chown build:build /home/build /work
392024-02-15T02:56:49.265Z+ chmod 0700 /home/build /work
402024-02-15T02:56:49.273Zprocess exited: duration 4412 ms, exit code 0
 
412024-02-15T02:56:49.284Zstarting task 1: "authentication"
422024-02-15T02:56:49.313Zprocess exited: duration 28 ms, exit code 0
 
432024-02-15T02:56:49.760Zstarting task 2: "build"
442024-02-15T02:56:49.775Z+ banner cores
452024-02-15T02:56:49.775Z
462024-02-15T02:56:49.775Z #### #### ##### ###### ####
472024-02-15T02:56:49.775Z # # # # # # # #
482024-02-15T02:56:49.775Z # # # # # ##### ####
492024-02-15T02:56:49.775Z # # # ##### # #
502024-02-15T02:56:49.775Z # # # # # # # # #
512024-02-15T02:56:49.776Z #### #### # # ###### ####
522024-02-15T02:56:49.776Z
532024-02-15T02:56:49.776Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-15T02:56:49.776Z+ echo 'input bins dir contains:'
552024-02-15T02:56:49.776Zinput bins dir contains:
562024-02-15T02:56:49.776Z+ ls -ltr /input/build/work/bins
572024-02-15T02:56:49.776Ztotal 760543
582024-02-15T02:56:49.777Z-rw-r--r-- 1 root root 121382752 Feb 15 02:55 crucible-downstairs.gz
592024-02-15T02:56:49.777Z-rw-r--r-- 1 root root 91080250 Feb 15 02:55 crucible-hammer.gz
602024-02-15T02:56:49.777Z-rw-r--r-- 1 root root 104713354 Feb 15 02:56 crutest.gz
612024-02-15T02:56:49.777Z-rw-r--r-- 1 root root 71941606 Feb 15 02:56 dsc.gz
622024-02-15T02:56:49.777Z+ echo 'input script dir contains:'
632024-02-15T02:56:49.777Zinput script dir contains:
642024-02-15T02:56:49.777Z+ ls -ltr /input/build/work/scripts
652024-02-15T02:56:49.777Ztotal 38
662024-02-15T02:56:49.777Z-rw-r--r-- 1 root root 3056 Feb 15 02:56 perf-downstairs-tick.d
672024-02-15T02:56:49.778Z-rw-r--r-- 1 root root 939 Feb 15 02:56 test_ds.sh
682024-02-15T02:56:49.778Z-rw-r--r-- 1 root root 3746 Feb 15 02:56 test_live_repair.sh
692024-02-15T02:56:49.778Z-rw-r--r-- 1 root root 6416 Feb 15 02:56 test_repair.sh
702024-02-15T02:56:49.778Z-rw-r--r-- 1 root root 2969 Feb 15 02:56 test_replay.sh
712024-02-15T02:56:49.778Z-rw-r--r-- 1 root root 9465 Feb 15 02:56 test_up.sh
722024-02-15T02:56:49.778Z-rw-r--r-- 1 root root 2824 Feb 15 02:56 upstairs_info.d
732024-02-15T02:56:49.779Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-15T02:56:49.779Z+ echo ' chmod input script dir contains:'
752024-02-15T02:56:49.779Z chmod input script dir contains:
762024-02-15T02:56:49.779Z+ ls -ltr /input/build/work/scripts
772024-02-15T02:56:49.779Ztotal 38
782024-02-15T02:56:49.779Z-rwxr-xr-x 1 root root 3056 Feb 15 02:56 perf-downstairs-tick.d
792024-02-15T02:56:49.780Z-rwxr-xr-x 1 root root 939 Feb 15 02:56 test_ds.sh
802024-02-15T02:56:49.780Z-rwxr-xr-x 1 root root 3746 Feb 15 02:56 test_live_repair.sh
812024-02-15T02:56:49.780Z-rwxr-xr-x 1 root root 6416 Feb 15 02:56 test_repair.sh
822024-02-15T02:56:49.780Z-rwxr-xr-x 1 root root 2969 Feb 15 02:56 test_replay.sh
832024-02-15T02:56:49.780Z-rwxr-xr-x 1 root root 9465 Feb 15 02:56 test_up.sh
842024-02-15T02:56:49.781Z-rwxr-xr-x 1 root root 2824 Feb 15 02:56 upstairs_info.d
852024-02-15T02:56:49.781Z+ banner unpack
862024-02-15T02:56:49.781Z
872024-02-15T02:56:49.781Z # # # # ##### ## #### # #
882024-02-15T02:56:49.781Z # # ## # # # # # # # # #
892024-02-15T02:56:49.781Z # # # # # # # # # # ####
902024-02-15T02:56:49.782Z # # # # # ##### ###### # # #
912024-02-15T02:56:49.782Z # # # ## # # # # # # #
922024-02-15T02:56:49.782Z #### # # # # # #### # #
932024-02-15T02:56:49.782Z
942024-02-15T02:56:49.782Z+ mkdir -p /var/tmp/bins
952024-02-15T02:56:49.782Z+ for t in "$input/bins/"*.gz
962024-02-15T02:56:49.782Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-15T02:56:49.783Z+ b=crucible-downstairs.gz
982024-02-15T02:56:49.783Z+ b=crucible-downstairs
992024-02-15T02:56:49.783Z+ gunzip
1002024-02-15T02:56:52.297Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-15T02:56:52.308Z+ for t in "$input/bins/"*.gz
1022024-02-15T02:56:52.308Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-15T02:56:52.308Z+ b=crucible-hammer.gz
1042024-02-15T02:56:52.308Z+ b=crucible-hammer
1052024-02-15T02:56:52.308Z+ gunzip
1062024-02-15T02:56:54.350Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-15T02:56:54.353Z+ for t in "$input/bins/"*.gz
1082024-02-15T02:56:54.356Z++ basename /input/build/work/bins/crutest.gz
1092024-02-15T02:56:54.356Z+ b=crutest.gz
1102024-02-15T02:56:54.356Z+ b=crutest
1112024-02-15T02:56:54.356Z+ gunzip
1122024-02-15T02:56:56.823Z+ chmod +x /var/tmp/bins/crutest
1132024-02-15T02:56:56.827Z+ for t in "$input/bins/"*.gz
1142024-02-15T02:56:56.827Z++ basename /input/build/work/bins/dsc.gz
1152024-02-15T02:56:56.827Z+ b=dsc.gz
1162024-02-15T02:56:56.827Z+ b=dsc
1172024-02-15T02:56:56.827Z+ gunzip
1182024-02-15T02:56:58.460Z+ chmod +x /var/tmp/bins/dsc
1192024-02-15T02:56:58.925Z+ export BINDIR=/var/tmp/bins
1202024-02-15T02:56:58.925Z+ BINDIR=/var/tmp/bins
1212024-02-15T02:56:58.925Z+ banner repair
1222024-02-15T02:56:58.925Z
1232024-02-15T02:56:58.925Z ##### ###### ##### ## # #####
1242024-02-15T02:56:58.925Z # # # # # # # # # #
1252024-02-15T02:56:58.926Z # # ##### # # # # # # #
1262024-02-15T02:56:58.926Z ##### # ##### ###### # #####
1272024-02-15T02:56:58.926Z # # # # # # # # #
1282024-02-15T02:56:58.926Z # # ###### # # # # # #
1292024-02-15T02:56:58.926Z
1302024-02-15T02:56:58.926Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-15T02:56:58.926ZTurn off color for downstairs dump
1322024-02-15T02:56:58.926ZCreating region directory at: /var/tmp/test_repair
1332024-02-15T02:56:58.926ZCreating dsc directory at: /tmp/dsc
1342024-02-15T02:56:58.926ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.081394866
1352024-02-15T02:56:58.927ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.04338743
1362024-02-15T02:56:58.927ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.042510662
1372024-02-15T02:56:58.927ZRegion set with 3 regions was created
1382024-02-15T02:56:58.927Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-15T02:56:58.927ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-15T02:56:58.927Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.823434448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141}
1412024-02-15T02:56:58.927Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824198122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141}
1422024-02-15T02:56:58.927Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824224778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141}
1432024-02-15T02:56:58.927Z{"msg":"Crucible 499a3386-42de-4a05-8cc8-e62bc182d601 has session id: 6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824247046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1442024-02-15T02:56:58.927Z{"msg":"Upstairs opts: Upstairs UUID: 499a3386-42de-4a05-8cc8-e62bc182d601, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824276442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1452024-02-15T02:56:58.927Z{"msg":"Crucible stats registered with UUID: 499a3386-42de-4a05-8cc8-e62bc182d601","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824300229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1462024-02-15T02:56:58.928ZCrucible runtime is spawned
1472024-02-15T02:56:58.928Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824712917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141}
1482024-02-15T02:56:58.928Z{"msg":"499a3386-42de-4a05-8cc8-e62bc182d601 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824808096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1492024-02-15T02:56:58.928Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824860809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1502024-02-15T02:56:58.928Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824910323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1512024-02-15T02:56:58.928Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824944728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1522024-02-15T02:56:58.928Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.824981114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1532024-02-15T02:56:58.928Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825041026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1542024-02-15T02:56:58.928Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825067233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1552024-02-15T02:56:58.928Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825106598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1562024-02-15T02:56:58.928Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825146163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1572024-02-15T02:56:58.928Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825175539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1582024-02-15T02:56:58.929Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825431877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1592024-02-15T02:56:58.929Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825537214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1602024-02-15T02:56:58.929Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.825586118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1612024-02-15T02:56:58.929Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.82708585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1622024-02-15T02:56:58.929Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.827133884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1632024-02-15T02:56:58.929Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.827150992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1642024-02-15T02:56:58.929Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.827177788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1652024-02-15T02:56:58.929Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.827192157Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1662024-02-15T02:56:58.929Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.827206525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1672024-02-15T02:56:58.929Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.828347802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1682024-02-15T02:56:58.930Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.828391176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1692024-02-15T02:56:58.930Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.828408914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1702024-02-15T02:56:58.930Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829317251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1712024-02-15T02:56:58.930Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829366594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1722024-02-15T02:56:58.930Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829387142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1732024-02-15T02:56:58.930Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829408569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1742024-02-15T02:56:58.931Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829423337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1752024-02-15T02:56:58.931Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829443735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1762024-02-15T02:56:58.931Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829456493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1772024-02-15T02:56:58.931Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829470451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1782024-02-15T02:56:58.931Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829592016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1792024-02-15T02:56:58.931Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829619143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1802024-02-15T02:56:58.932Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829786902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1812024-02-15T02:56:58.932Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829813008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1822024-02-15T02:56:58.932Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829827536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1832024-02-15T02:56:58.932Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829848924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1842024-02-15T02:56:58.932Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829863352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1852024-02-15T02:56:58.932Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.82988322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1862024-02-15T02:56:58.933Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829897928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1872024-02-15T02:56:58.933Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829917435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1882024-02-15T02:56:58.933Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829931174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1892024-02-15T02:56:58.933Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829952211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1902024-02-15T02:56:58.933Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829966749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1912024-02-15T02:56:58.933Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.829988426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1922024-02-15T02:56:58.934Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.830000905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1932024-02-15T02:56:58.934Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.830014503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1942024-02-15T02:56:58.934Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.830034341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1952024-02-15T02:56:58.934Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1962024-02-15T02:56:58.935Z,"time":"2024-02-15T02:56:57.830047879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1972024-02-15T02:56:58.935Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.830073946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1982024-02-15T02:56:58.935Z{"msg":"499a3386-42de-4a05-8cc8-e62bc182d601 is now active with session: 6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.830087424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
1992024-02-15T02:56:58.935Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.830106552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141,"session_id":"6315ef4e-d719-4ba0-9d5d-52cfc0c3f2ad"}
2002024-02-15T02:56:58.935Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:56:57.83011932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141}
2012024-02-15T02:56:58.936ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-15T02:56:58.936ZFill test
2032024-02-15T02:56:59.200ZRead and Verify all blocks (0..600 range:false)
2042024-02-15T02:56:59.271ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-15T02:56:59.272ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-15T02:57:00.498Z----------------------------------------------------------------
2072024-02-15T02:57:00.498Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-15T02:57:00.498ZStates: Active Active Active
2092024-02-15T02:57:00.498ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-15T02:57:00.498Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-15T02:57:00.498Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-15T02:57:00.498Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-15T02:57:00.498Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-15T02:57:00.498Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-15T02:57:00.498Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-15T02:57:00.498Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-15T02:57:00.498Z New 0 0 0 0
2182024-02-15T02:57:00.498Z Sent 0 0 0 0
2192024-02-15T02:57:00.498Z Done 6 6 6 18
2202024-02-15T02:57:00.498Z Skipped 0 0 0 0
2212024-02-15T02:57:00.498Z Error 0 0 0 0
2222024-02-15T02:57:00.498ZLast Flush: 1006 1006 1006
2232024-02-15T02:57:00.498ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-15T02:57:00.499ZUpstairs last five completed: 13 12 11 10 9
2252024-02-15T02:57:00.499ZCLIENT: Up:0 ds:6 act:3
2262024-02-15T02:57:03.270Z----------------------------------------------------------------
2272024-02-15T02:57:03.270Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-15T02:57:03.270ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-15T02:57:03.271ZUpstairs last five completed: 14 13 12 11 10
2302024-02-15T02:57:03.271ZCLIENT: Up:0 ds:0 act:3
2312024-02-15T02:57:03.271ZCLIENT: All crucible jobs finished, exiting program
2322024-02-15T02:57:03.296Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:02.226889438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1141}
2332024-02-15T02:57:03.297Z
2342024-02-15T02:57:03.297ZBegin loop 1
2352024-02-15T02:57:03.297ZDownstairs to restart: 2
2362024-02-15T02:57:03.306Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254193636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143}
2372024-02-15T02:57:03.306Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254547562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143}
2382024-02-15T02:57:03.306Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
2392024-02-15T02:57:03.307Z,"time":"2024-02-15T02:57:02.254582958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143}
2402024-02-15T02:57:03.307Z{"msg":"Crucible 09682cd1-c08d-45b6-8ddd-8dc43c5a9bbf has session id: 7f91413d-646e-4407-849a-be55c23104cb","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254628992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2412024-02-15T02:57:03.307Z{"msg":"Upstairs opts: Upstairs UUID: 09682cd1-c08d-45b6-8ddd-8dc43c5a9bbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254671037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2422024-02-15T02:57:03.307Z{"msg":"Crucible stats registered with UUID: 09682cd1-c08d-45b6-8ddd-8dc43c5a9bbf","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254702163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2432024-02-15T02:57:03.307Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254716471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143}
2442024-02-15T02:57:03.308Z{"msg":"09682cd1-c08d-45b6-8ddd-8dc43c5a9bbf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254788962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2452024-02-15T02:57:03.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254815789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2462024-02-15T02:57:03.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254831397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2472024-02-15T02:57:03.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254853964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2482024-02-15T02:57:03.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.25488205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2492024-02-15T02:57:03.308Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254929804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2502024-02-15T02:57:03.308Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254958391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2512024-02-15T02:57:03.308Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.254996416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2522024-02-15T02:57:03.309Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.255029872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2532024-02-15T02:57:03.309Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.255056079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2542024-02-15T02:57:03.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.255376708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2552024-02-15T02:57:03.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.255449079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2562024-02-15T02:57:03.309Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:02.255511741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2572024-02-15T02:57:03.309Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:02.255549777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2582024-02-15T02:57:03.309Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:02.255639026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2592024-02-15T02:57:03.309Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.255668232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2602024-02-15T02:57:03.309Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.25568361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2612024-02-15T02:57:03.309Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.255807554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2622024-02-15T02:57:03.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.256261577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2632024-02-15T02:57:03.310Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.256380762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2642024-02-15T02:57:03.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.256419977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2652024-02-15T02:57:03.310Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.256446774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2662024-02-15T02:57:03.310Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.256813708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2672024-02-15T02:57:03.310Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.256929264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2682024-02-15T02:57:03.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.257302317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2692024-02-15T02:57:03.310Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.257334913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2702024-02-15T02:57:03.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.25736042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2712024-02-15T02:57:03.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.257490114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2722024-02-15T02:57:03.310Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.25752173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2732024-02-15T02:57:03.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:02.257538587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2742024-02-15T02:57:04.298Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.257223859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2752024-02-15T02:57:04.298Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.257305749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2762024-02-15T02:57:04.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.258200247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2772024-02-15T02:57:04.304Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.258272058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2782024-02-15T02:57:04.304Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.258775515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2792024-02-15T02:57:04.304Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259349813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2802024-02-15T02:57:04.304Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259391788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2812024-02-15T02:57:04.304Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259411005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2822024-02-15T02:57:04.304Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259460039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2832024-02-15T02:57:04.304Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259495085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2842024-02-15T02:57:04.304Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259527231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2852024-02-15T02:57:04.304Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2862024-02-15T02:57:04.304Z,"time":"2024-02-15T02:57:03.259569796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2872024-02-15T02:57:04.305Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259602692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2882024-02-15T02:57:04.305Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259637657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2892024-02-15T02:57:04.305Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259667933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2902024-02-15T02:57:04.305Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.25969654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2912024-02-15T02:57:04.305Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259722797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2922024-02-15T02:57:04.305Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259760592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2932024-02-15T02:57:04.305Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259789898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2942024-02-15T02:57:04.305Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259826213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2952024-02-15T02:57:04.305Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.2598558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2962024-02-15T02:57:04.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259890885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2972024-02-15T02:57:04.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259920812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2982024-02-15T02:57:04.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.259956777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
2992024-02-15T02:57:04.305Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3002024-02-15T02:57:04.305Z,"time":"2024-02-15T02:57:03.259987213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
3012024-02-15T02:57:04.306Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.260019129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
3022024-02-15T02:57:04.306Z{"msg":"09682cd1-c08d-45b6-8ddd-8dc43c5a9bbf is now active with session: 7f91413d-646e-4407-849a-be55c23104cb","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.260055735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
3032024-02-15T02:57:04.306Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.260072493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143,"session_id":"7f91413d-646e-4407-849a-be55c23104cb"}
3042024-02-15T02:57:04.306Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:03.260108888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143}
3052024-02-15T02:57:04.306ZLoading write count information from file "/tmp/test_repair_verify.data"
3062024-02-15T02:57:04.306ZRun Repair workload
3072024-02-15T02:57:04.306Z01/30 Write block 10 len 12288 data: 2 2 2
3082024-02-15T02:57:04.313Z02/30 Read block 438 len 8192
3092024-02-15T02:57:04.313Z03/30 Write block 335 len 24576 data: 2 2 2 2 2 2
3102024-02-15T02:57:04.313Z04/30 Write block 323 len 16384 data: 2 2 2 2
3112024-02-15T02:57:04.313Z05/30 Write block 365 len 4096 data: 2
3122024-02-15T02:57:04.313Z06/30 Write block 532 len 20480 data: 2 2 2 2 2
3132024-02-15T02:57:04.313Z07/30 Write block 305 len 24576 data: 2 2 2 2 2 2
3142024-02-15T02:57:04.313Z08/30 Read block 282 len 24576
3152024-02-15T02:57:04.314Z09/30 Write block 512 len 20480 data: 2 2 2 2 2
3162024-02-15T02:57:04.314Z10/30 Write block 556 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3172024-02-15T02:57:04.314Z11/30 Write block 226 len 16384 data: 2 2 2 2
3182024-02-15T02:57:04.314Z12/30 Write block 426 len 28672 data: 2 2 2 2 2 2 2
3192024-02-15T02:57:04.329Z13/30 Flush
3202024-02-15T02:57:04.370Z14/30 Write block 555 len 12288 data: 2 3 3
3212024-02-15T02:57:04.374Z15/30 Read block 80 len 24576
3222024-02-15T02:57:04.374Z16/30 Read block 77 len 16384
3232024-02-15T02:57:04.374Z17/30 Write block 594 len 4096 data: 2
3242024-02-15T02:57:04.375Z18/30 Read block 252 len 4096
3252024-02-15T02:57:04.375Z19/30 Write block 468 len 16384 data: 2 2 2 2
3262024-02-15T02:57:04.378Z20/30 Read block 34 len 36864
3272024-02-15T02:57:04.378Z21/30 Read block 192 len 40960
3282024-02-15T02:57:04.378Z22/30 Write block 304 len 16384 data: 2 3 3 3
3292024-02-15T02:57:04.378Z23/30 Read block 156 len 28672
3302024-02-15T02:57:04.378Z24/30 Read block 457 len 32768
3312024-02-15T02:57:04.382Z25/30 Read block 305 len 4096
3322024-02-15T02:57:04.382Z26/30 Write block 125 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3332024-02-15T02:57:04.382Z27/30 Read block 207 len 32768
3342024-02-15T02:57:04.385Z28/30 Write block 88 len 28672 data: 2 2 2 2 2 2 2
3352024-02-15T02:57:04.385Z29/30 Write block 427 len 28672 data: 3 3 3 3 3 3 2
3362024-02-15T02:57:04.385Z30/30 Write block 318 len 12288 data: 2 2 2
3372024-02-15T02:57:04.386Z----------------------------------------------------------------
3382024-02-15T02:57:04.386Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3392024-02-15T02:57:04.386ZStates: Active Active Active
3402024-02-15T02:57:04.386ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3412024-02-15T02:57:04.386Z 1 Acked 1000 Write 3 Done Done Done false
3422024-02-15T02:57:04.386Z 2 Acked 1001 Read 2 Done Done Done false
3432024-02-15T02:57:04.386Z 3 Acked 1002 Write 6 Done Done Done false
3442024-02-15T02:57:04.386Z 4 Acked 1003 Write 4 Done Done Done false
3452024-02-15T02:57:04.386Z 5 Acked 1004 Write 1 Done Done Sent false
3462024-02-15T02:57:04.386Z 6 Acked 1005 Write 5 Done Done Sent false
3472024-02-15T02:57:04.386Z 7 Acked 1006 Write 6 Done Done Sent false
3482024-02-15T02:57:04.387Z 8 Acked 1007 Read 6 Done Done Sent false
3492024-02-15T02:57:04.387Z 9 Acked 1008 Write 5 Done Done Sent false
3502024-02-15T02:57:04.387Z 10 Acked 1009 Write 10 Done Done Sent false
3512024-02-15T02:57:04.387Z 11 Acked 1010 Write 4 Done Done Sent false
3522024-02-15T02:57:04.387Z 12 Acked 1011 Write 7 Done Done Sent false
3532024-02-15T02:57:04.387Z 13 Acked 1012 Flush 0 Done Done Sent false
3542024-02-15T02:57:04.387Z 14 Acked 1013 Write 3 Done Done Sent false
3552024-02-15T02:57:04.387Z 15 Acked 1014 Read 6 Done Done Sent false
3562024-02-15T02:57:04.387Z 16 Acked 1015 Read 4 Done Done Sent false
3572024-02-15T02:57:04.387Z 17 Acked 1016 Write 1 Done Done Sent false
3582024-02-15T02:57:04.387Z 18 Acked 1017 Read 1 Done Done Sent false
3592024-02-15T02:57:04.387Z 19 Acked 1018 Write 4 Done Done Sent false
3602024-02-15T02:57:04.387Z 20 Acked 1019 Read 9 Done Done Sent false
3612024-02-15T02:57:04.387Z 21 Acked 1020 Read 10 Done Done Sent false
3622024-02-15T02:57:04.387Z 22 Acked 1021 Write 4 Done Done Sent false
3632024-02-15T02:57:04.387Z 23 Acked 1022 Read 7 Done Done Sent false
3642024-02-15T02:57:04.387Z 24 Acked 1023 Read 8 Done Done Sent false
3652024-02-15T02:57:04.387Z 25 Acked 1024 Read 1 Done Done Sent false
3662024-02-15T02:57:04.387Z 26 Acked 1025 Write 10 Done Done Sent false
3672024-02-15T02:57:04.387Z 27 Acked 1026 Read 8 Done Done Sent false
3682024-02-15T02:57:04.387Z 28 Acked 1027 Write 7 Done Done Sent false
3692024-02-15T02:57:04.388Z 29 Acked 1028 Write 7 Sent Sent Sent false
3702024-02-15T02:57:04.388Z 30 Acked 1029 Write 3 Sent Sent Sent false
3712024-02-15T02:57:04.388Z STATES DS:0 DS:1 DS:2 TOTAL
3722024-02-15T02:57:04.388Z New 0 0 0 0
3732024-02-15T02:57:04.388Z Sent 2 2 26 30
3742024-02-15T02:57:04.388Z Done 28 28 4 60
3752024-02-15T02:57:04.388Z Skipped 0 0 0 0
3762024-02-15T02:57:04.388Z Error 0 0 0 0
3772024-02-15T02:57:04.388ZLast Flush: 1012 1012 0
3782024-02-15T02:57:04.388ZDownstairs last five completed:
3792024-02-15T02:57:04.388ZUpstairs last five completed: 30 29 28 27 26
3802024-02-15T02:57:04.388Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:03.344381709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1143}
3812024-02-15T02:57:04.388ZWrote out file "/tmp/test_repair_verify.data"
3822024-02-15T02:57:04.413Z
3832024-02-15T02:57:04.414ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3842024-02-15T02:57:04.414ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3852024-02-15T02:57:04.414Z 0 000-019 2 2 1 2 2 1 F F T <---
3862024-02-15T02:57:04.418Z 1 020-039 1 1 1 1 1 1 F F F
3872024-02-15T02:57:04.418Z 2 040-059 1 1 1 1 1 1 F F F
3882024-02-15T02:57:04.419Z 3 060-079 1 1 1 1 1 1 F F F
3892024-02-15T02:57:04.419Z 4 080-099 1 1 1 1 1 1 T T F <---
3902024-02-15T02:57:04.419Z 5 100-119 1 1 1 1 1 1 F F F
3912024-02-15T02:57:04.419Z 6 120-139 1 1 1 1 1 1 T T F <---
3922024-02-15T02:57:04.419Z 7 140-159 1 1 1 1 1 1 F F F
3932024-02-15T02:57:04.419Z 8 160-179 1 1 1 1 1 1 F F F
3942024-02-15T02:57:04.419Z 9 180-199 1 1 1 1 1 1 F F F
3952024-02-15T02:57:04.419Z 10 200-219 1 1 1 1 1 1 F F F
3962024-02-15T02:57:04.419Z 11 220-239 2 2 1 2 2 1 F F F <---
3972024-02-15T02:57:04.419Z 12 240-259 1 1 1 1 1 1 F F F
3982024-02-15T02:57:04.419Z 13 260-279 1 1 1 1 1 1 F F F
3992024-02-15T02:57:04.419Z 14 280-299 1 1 1 1 1 1 F F F
4002024-02-15T02:57:04.420Z 15 300-319 2 2 1 2 2 1 T T F <---
4012024-02-15T02:57:04.420Z 16 320-339 2 2 1 2 2 1 T T T <---
4022024-02-15T02:57:04.420Z 17 340-359 2 2 1 2 2 1 F F T <---
4032024-02-15T02:57:04.420Z 18 360-379 2 2 1 2 2 1 F F F <---
4042024-02-15T02:57:04.420Z 19 380-399 1 1 1 1 1 1 F F F
4052024-02-15T02:57:04.420Z 20 400-419 1 1 1 1 1 1 F F F
4062024-02-15T02:57:04.420Z 21 420-439 2 2 1 2 2 1 T T F <---
4072024-02-15T02:57:04.420Z 22 440-459 1 1 1 1 1 1 F F F
4082024-02-15T02:57:04.420Z 23 460-479 1 1 1 1 1 1 T T F <---
4092024-02-15T02:57:04.420Z 24 480-499 1 1 1 1 1 1 F F F
4102024-02-15T02:57:04.420Z 25 500-519 2 2 1 2 2 1 F F F <---
4112024-02-15T02:57:04.420Z 26 520-539 2 2 1 2 2 1 F F F <---
4122024-02-15T02:57:04.420Z 27 540-559 2 2 1 2 2 1 T T F <---
4132024-02-15T02:57:04.420Z 28 560-579 2 2 1 2 2 1 F F F <---
4142024-02-15T02:57:04.420Z 29 580-599 1 1 1 1 1 1 T T F <---
4152024-02-15T02:57:04.420ZMax gen: 2, Max flush: 2
4162024-02-15T02:57:04.421ZError: Difference in extent metadata found!
4172024-02-15T02:57:04.421ZA Difference in extent metadata is expected here
4182024-02-15T02:57:04.421ZOn loop 1
4192024-02-15T02:57:04.421Z
4202024-02-15T02:57:04.421ZVerifying data now
4212024-02-15T02:57:11.624ZNo change after verify
4222024-02-15T02:57:11.624ZLoop: 1 Downstairs dump after verify (and repair):
4232024-02-15T02:57:11.637ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4242024-02-15T02:57:11.637Z 0 000-019 2 2 2 2 2 2 F F F
4252024-02-15T02:57:11.637Z 1 020-039 1 1 1 1 1 1 F F F
4262024-02-15T02:57:11.637Z 2 040-059 1 1 1 1 1 1 F F F
4272024-02-15T02:57:11.637Z 3 060-079 1 1 1 1 1 1 F F F
4282024-02-15T02:57:11.637Z 4 080-099 3 3 3 3 3 3 F F F
4292024-02-15T02:57:11.637Z 5 100-119 1 1 1 1 1 1 F F F
4302024-02-15T02:57:11.637Z 6 120-139 3 3 3 3 3 3 F F F
4312024-02-15T02:57:11.640Z 7 140-159 1 1 1 1 1 1 F F F
4322024-02-15T02:57:11.640Z 8 160-179 1 1 1 1 1 1 F F F
4332024-02-15T02:57:11.640Z 9 180-199 1 1 1 1 1 1 F F F
4342024-02-15T02:57:11.640Z 10 200-219 1 1 1 1 1 1 F F F
4352024-02-15T02:57:11.640Z 11 220-239 2 2 2 2 2 2 F F F
4362024-02-15T02:57:11.640Z 12 240-259 1 1 1 1 1 1 F F F
4372024-02-15T02:57:11.641Z 13 260-279 1 1 1 1 1 1 F F F
4382024-02-15T02:57:11.641Z 14 280-299 1 1 1 1 1 1 F F F
4392024-02-15T02:57:11.641Z 15 300-319 3 3 3 3 3 3 F F F
4402024-02-15T02:57:11.641Z 16 320-339 3 3 3 3 3 3 F F F
4412024-02-15T02:57:11.641Z 17 340-359 2 2 2 2 2 2 F F F
4422024-02-15T02:57:11.641Z 18 360-379 2 2 2 2 2 2 F F F
4432024-02-15T02:57:11.641Z 19 380-399 1 1 1 1 1 1 F F F
4442024-02-15T02:57:11.641Z 20 400-419 1 1 1 1 1 1 F F F
4452024-02-15T02:57:11.641Z 21 420-439 3 3 3 3 3 3 F F F
4462024-02-15T02:57:11.641Z 22 440-459 1 1 1 1 1 1 F F F
4472024-02-15T02:57:11.641Z 23 460-479 3 3 3 3 3 3 F F F
4482024-02-15T02:57:11.641Z 24 480-499 1 1 1 1 1 1 F F F
4492024-02-15T02:57:11.642Z 25 500-519 2 2 2 2 2 2 F F F
4502024-02-15T02:57:11.642Z 26 520-539 2 2 2 2 2 2 F F F
4512024-02-15T02:57:11.642Z 27 540-559 3 3 3 3 3 3 F F F
4522024-02-15T02:57:11.642Z 28 560-579 2 2 2 2 2 2 F F F
4532024-02-15T02:57:11.642Z 29 580-599 3 3 3 3 3 3 F F F
4542024-02-15T02:57:11.642ZMax gen: 3, Max flush: 3
4552024-02-15T02:57:11.642Z
4562024-02-15T02:57:11.642ZBegin loop 2
4572024-02-15T02:57:11.642ZDownstairs to restart: 0
4582024-02-15T02:57:11.668Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.600476772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151}
4592024-02-15T02:57:11.668Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000ZCrucible runtime is spawned
4602024-02-15T02:57:11.668Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.60081132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151}
4612024-02-15T02:57:11.669Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.600858894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151}
4622024-02-15T02:57:11.669Z{"msg":"Crucible 3da179e9-074c-4661-98d0-6d27d5532be4 has session id: 72ad2810-0d38-4b68-8271-f480f4ce445c","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.60088672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4632024-02-15T02:57:11.669Z{"msg":"Upstairs opts: Upstairs UUID: 3da179e9-074c-4661-98d0-6d27d5532be4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.600940924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4642024-02-15T02:57:11.669Z{"msg":"Crucible stats registered with UUID: 3da179e9-074c-4661-98d0-6d27d5532be4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.60097034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4652024-02-15T02:57:11.669Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601025933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151}
4662024-02-15T02:57:11.669Z{"msg":"3da179e9-074c-4661-98d0-6d27d5532be4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.60105269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4672024-02-15T02:57:11.669Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601090315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4682024-02-15T02:57:11.669Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601119721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4692024-02-15T02:57:11.669Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601201111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4702024-02-15T02:57:11.669Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601271302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4712024-02-15T02:57:11.669Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601333784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4722024-02-15T02:57:11.669Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.60136721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4732024-02-15T02:57:11.669Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601400046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4742024-02-15T02:57:11.669Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601435342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4752024-02-15T02:57:11.670Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601484286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4762024-02-15T02:57:11.670Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.60152624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4772024-02-15T02:57:11.670Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:10.601558287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4782024-02-15T02:57:11.670Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:10.601599701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4792024-02-15T02:57:11.670Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:10.601645016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4802024-02-15T02:57:11.670Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.6016866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4812024-02-15T02:57:11.670Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601723926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4822024-02-15T02:57:11.670Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601746383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4832024-02-15T02:57:11.670Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.601784378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4842024-02-15T02:57:11.670Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.602157551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4852024-02-15T02:57:11.670Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.602196556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4862024-02-15T02:57:11.670Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.602213814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4872024-02-15T02:57:11.670Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.602238301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4882024-02-15T02:57:11.670Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.602803491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4892024-02-15T02:57:11.670Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.602961291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4902024-02-15T02:57:11.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.603412424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4912024-02-15T02:57:11.670Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.60344784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4922024-02-15T02:57:11.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.603501943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4932024-02-15T02:57:11.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.603532129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4942024-02-15T02:57:11.670Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.603547307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4952024-02-15T02:57:11.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:10.603564515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4962024-02-15T02:57:12.652Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.602135227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4972024-02-15T02:57:12.652Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.602222597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4982024-02-15T02:57:12.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.603101726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
4992024-02-15T02:57:12.668Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.603141061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5002024-02-15T02:57:12.668Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.603680184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5012024-02-15T02:57:12.668Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604294147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5022024-02-15T02:57:12.669Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604321813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5032024-02-15T02:57:12.669Z{"msg":"[0]R flush_numbers[0..12]: [2, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604336982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5042024-02-15T02:57:12.669Z{"msg":"[0]R generation[0..12]: [2, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.60435139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5052024-02-15T02:57:12.669Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604374567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5062024-02-15T02:57:12.669Z{"msg":"[1]R flush_numbers[0..12]: [2, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604389345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5072024-02-15T02:57:12.669Z{"msg":"[1]R generation[0..12]: [2, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604403003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5082024-02-15T02:57:12.669Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604417212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5092024-02-15T02:57:12.669Z{"msg":"[2]R flush_numbers[0..12]: [2, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604438599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5102024-02-15T02:57:12.670Z{"msg":"[2]R generation[0..12]: [2, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604452847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5112024-02-15T02:57:12.670Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604473914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5122024-02-15T02:57:12.670Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604487413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5132024-02-15T02:57:12.670Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604514929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5142024-02-15T02:57:12.670Z{"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604528878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5152024-02-15T02:57:12.670Z{"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
5162024-02-15T02:57:12.670Z,"time":"2024-02-15T02:57:11.604542336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5172024-02-15T02:57:12.670Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604558954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5182024-02-15T02:57:12.670Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604571452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5192024-02-15T02:57:12.670Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.60459227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5202024-02-15T02:57:12.670Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604607108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5212024-02-15T02:57:12.670Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604633645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5222024-02-15T02:57:12.671Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604646523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5232024-02-15T02:57:12.671Z{"msg":"3da179e9-074c-4661-98d0-6d27d5532be4 is now active with session: 72ad2810-0d38-4b68-8271-f480f4ce445c","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604659741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5242024-02-15T02:57:12.671Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604683418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151,"session_id":"72ad2810-0d38-4b68-8271-f480f4ce445c"}
5252024-02-15T02:57:12.671Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:11.604696257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151}
5262024-02-15T02:57:12.671ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5272024-02-15T02:57:12.671ZLoading write count information from file "/tmp/test_repair_verify.data"
5282024-02-15T02:57:12.671ZRun Repair workload
5292024-02-15T02:57:12.671Z01/30 Write block 64 len 12288 data: 2 2 2
5302024-02-15T02:57:12.671Z02/30 Flush
5312024-02-15T02:57:12.671Z03/30 Write block 3 len 32768 data: 2 2 2 2 2 2 2 3
5322024-02-15T02:57:12.672Z04/30 Write block 306 len 28672 data: 4 4 3 3 3 2 2
5332024-02-15T02:57:12.672Z05/30 Write block 94 len 20480 data: 3 2 2 2 2
5342024-02-15T02:57:12.672Z06/30 Flush
5352024-02-15T02:57:12.672Z07/30 Write block 28 len 32768 data: 2 2 2 2 2 2 2 2
5362024-02-15T02:57:12.672Z08/30 Write block 303 len 12288 data: 2 3 4
5372024-02-15T02:57:12.672Z09/30 Write block 589 len 4096 data: 2
5382024-02-15T02:57:12.672Z10/30 Write block 104 len 24576 data: 2 2 2 2 2 2
5392024-02-15T02:57:12.672Z11/30 Flush
5402024-02-15T02:57:12.672Z12/30 Write block 537 len 24576 data: 2 2 2 2 2 2
5412024-02-15T02:57:12.672Z13/30 Read block 218 len 24576
5422024-02-15T02:57:12.708Z14/30 Write block 163 len 36864 data: 2 2 2 2 2 2 2 2 2
5432024-02-15T02:57:12.708Z15/30 Flush
5442024-02-15T02:57:12.709Z16/30 Write block 442 len 24576 data: 2 2 2 2 2 2
5452024-02-15T02:57:12.709Z17/30 Read block 78 len 12288
5462024-02-15T02:57:12.709Z18/30 Write block 57 len 36864 data: 2 2 2 2 2 2 2 3 3
5472024-02-15T02:57:12.709Z19/30 Read block 322 len 16384
5482024-02-15T02:57:12.709Z20/30 Write block 388 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5492024-02-15T02:57:12.709Z21/30 Read block 435 len 28672
5502024-02-15T02:57:12.709Z22/30 Read block 405 len 20480
5512024-02-15T02:57:12.709Z23/30 Write block 184 len 24576 data: 2 2 2 2 2 2
5522024-02-15T02:57:12.709Z24/30 Read block 389 len 32768
5532024-02-15T02:57:12.709Z25/30 Write block 75 len 12288 data: 2 2 2
5542024-02-15T02:57:12.709Z26/30 Write block 378 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5552024-02-15T02:57:12.709Z27/30 Read block 48 len 4096
5562024-02-15T02:57:12.710Z28/30 Write block 109 len 40960 data: 3 2 2 2 2 2 2 2 2 2
5572024-02-15T02:57:12.710Z29/30 Read block 211 len 20480
5582024-02-15T02:57:12.710Z30/30 Read block 70 len 28672
5592024-02-15T02:57:12.710Z----------------------------------------------------------------
5602024-02-15T02:57:12.710Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5612024-02-15T02:57:12.710ZStates: Active Active Active
5622024-02-15T02:57:12.710ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5632024-02-15T02:57:12.710Z 1 Acked 1000 Write 3 Done Done Done false
5642024-02-15T02:57:12.710Z 2 Acked 1001 Flush 0 Sent Done Done false
5652024-02-15T02:57:12.710Z 3 Acked 1002 Write 8 Sent Done Done false
5662024-02-15T02:57:12.710Z 4 Acked 1003 Write 7 Sent Done Done false
5672024-02-15T02:57:12.710Z 5 Acked 1004 Write 5 Sent Done Done false
5682024-02-15T02:57:12.710Z 6 Acked 1005 Flush 0 Sent Done Done false
5692024-02-15T02:57:12.710Z 7 Acked 1006 Write 8 Sent Done Done false
5702024-02-15T02:57:12.710Z 8 Acked 1007 Write 3 Sent Done Done false
5712024-02-15T02:57:12.710Z 9 Acked 1008 Write 1 Sent Done Done false
5722024-02-15T02:57:12.710Z 10 Acked 1009 Write 6 Sent Done Done false
5732024-02-15T02:57:12.710Z 11 Acked 1010 Flush 0 Sent Done Done false
5742024-02-15T02:57:12.710Z 12 Acked 1011 Write 6 Sent Done Done false
5752024-02-15T02:57:12.710Z 13 Acked 1012 Read 6 Sent Done Done false
5762024-02-15T02:57:12.711Z 14 Acked 1013 Write 9 Sent Done Done false
5772024-02-15T02:57:12.711Z 15 Acked 1014 Flush 0 Sent Done Done false
5782024-02-15T02:57:12.711Z 16 Acked 1015 Write 6 Sent Done Done false
5792024-02-15T02:57:12.711Z 17 Acked 1016 Read 3 Sent Done Done false
5802024-02-15T02:57:12.711Z 18 Acked 1017 Write 9 Sent Done Done false
5812024-02-15T02:57:12.711Z 19 Acked 1018 Read 4 Sent Done Done false
5822024-02-15T02:57:12.711Z 20 Acked 1019 Write 10 Sent Done Done false
5832024-02-15T02:57:12.711Z 21 Acked 1020 Read 7 Sent Done Done false
5842024-02-15T02:57:12.711Z 22 Acked 1021 Read 5 Sent Done Done false
5852024-02-15T02:57:12.711Z 23 Acked 1022 Write 6 Sent Done Done false
5862024-02-15T02:57:12.711Z 24 Acked 1023 Read 8 Sent Done Done false
5872024-02-15T02:57:12.711Z 25 Acked 1024 Write 3 Sent Done Done false
5882024-02-15T02:57:12.711Z 26 Acked 1025 Write 10 Sent Done Done false
5892024-02-15T02:57:12.711Z 27 Acked 1026 Read 1 Sent Done Done false
5902024-02-15T02:57:12.711Z 28 Acked 1027 Write 10 Sent Done Done false
5912024-02-15T02:57:12.711Z 29 Acked 1028 Read 5 Sent Done Done false
5922024-02-15T02:57:12.711Z 30 Acked 1029 Read 7 Sent Done Done false
5932024-02-15T02:57:12.711Z STATES DS:0 DS:1 DS:2 TOTAL
5942024-02-15T02:57:12.711Z New 0 0 0 0
5952024-02-15T02:57:12.711Z Sent 29 0 0 29
5962024-02-15T02:57:12.711Z Done 1 30 30 61
5972024-02-15T02:57:12.711Z Skipped 0 0 0 0
5982024-02-15T02:57:12.711Z Error 0 0 0 0
5992024-02-15T02:57:12.711ZLast Flush: 0 1014 1014
6002024-02-15T02:57:12.712ZDownstairs last five completed:
6012024-02-15T02:57:12.712ZUpstairs last five completed: 30 29 28 27 26
6022024-02-15T02:57:12.712Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:11.639342196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1151}
6032024-02-15T02:57:12.712ZWrote out file "/tmp/test_repair_verify.data"
6042024-02-15T02:57:12.712Z
6052024-02-15T02:57:12.712ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
6062024-02-15T02:57:12.723ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6072024-02-15T02:57:12.723Z 0 000-019 2 4 4 2 5 5 F F F <---
6082024-02-15T02:57:12.723Z 1 020-039 1 4 4 1 6 6 F F F <---
6092024-02-15T02:57:12.723Z 2 040-059 1 1 1 1 1 1 F T T <---
6102024-02-15T02:57:12.723Z 3 060-079 1 4 4 1 4 4 T T T <---
6112024-02-15T02:57:12.723Z 4 080-099 3 4 4 3 5 5 F F F <---
6122024-02-15T02:57:12.723Z 5 100-119 1 4 4 1 6 6 F T T <---
6132024-02-15T02:57:12.723Z 6 120-139 3 3 3 3 3 3 F F F
6142024-02-15T02:57:12.723Z 7 140-159 1 1 1 1 1 1 F F F
6152024-02-15T02:57:12.723Z 8 160-179 1 4 4 1 7 7 F F F <---
6162024-02-15T02:57:12.723Z 9 180-199 1 1 1 1 1 1 F T T <---
6172024-02-15T02:57:12.723Z 10 200-219 1 1 1 1 1 1 F F F
6182024-02-15T02:57:12.723Z 11 220-239 2 2 2 2 2 2 F F F
6192024-02-15T02:57:12.724Z 12 240-259 1 1 1 1 1 1 F F F
6202024-02-15T02:57:12.724Z 13 260-279 1 1 1 1 1 1 F F F
6212024-02-15T02:57:12.724Z 14 280-299 1 1 1 1 1 1 F F F
6222024-02-15T02:57:12.724Z 15 300-319 3 4 4 3 6 6 F F F <---
6232024-02-15T02:57:12.724Z 16 320-339 3 3 3 3 3 3 F F F
6242024-02-15T02:57:12.724Z 17 340-359 2 2 2 2 2 2 F F F
6252024-02-15T02:57:12.724Z 18 360-379 2 2 2 2 2 2 F T T <---
6262024-02-15T02:57:12.724Z 19 380-399 1 1 1 1 1 1 F T T <---
6272024-02-15T02:57:12.724Z 20 400-419 1 1 1 1 1 1 F F F
6282024-02-15T02:57:12.724Z 21 420-439 3 3 3 3 3 3 F F F
6292024-02-15T02:57:12.724Z 22 440-459 1 1 1 1 1 1 F T T <---
6302024-02-15T02:57:12.724Z 23 460-479 3 3 3 3 3 3 F F F
6312024-02-15T02:57:12.724Z 24 480-499 1 1 1 1 1 1 F F F
6322024-02-15T02:57:12.724Z 25 500-519 2 2 2 2 2 2 F F F
6332024-02-15T02:57:12.724Z 26 520-539 2 4 4 2 7 7 F F F <---
6342024-02-15T02:57:12.724Z 27 540-559 3 4 4 3 7 7 F F F <---
6352024-02-15T02:57:12.724Z 28 560-579 2 2 2 2 2 2 F F F
6362024-02-15T02:57:12.724Z 29 580-599 3 4 4 3 6 6 F F F <---
6372024-02-15T02:57:12.724ZMax gen: 4, Max flush: 7
6382024-02-15T02:57:12.724ZError: Difference in extent metadata found!
6392024-02-15T02:57:12.724ZA Difference in extent metadata is expected here
6402024-02-15T02:57:12.724ZOn loop 2
6412024-02-15T02:57:12.724Z
6422024-02-15T02:57:12.724ZVerifying data now
6432024-02-15T02:57:18.449ZNo change after verify
6442024-02-15T02:57:18.449ZLoop: 2 Downstairs dump after verify (and repair):
6452024-02-15T02:57:18.492ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6462024-02-15T02:57:18.492Z 0 000-019 4 4 4 5 5 5 F F F
6472024-02-15T02:57:18.492Z 1 020-039 4 4 4 6 6 6 F F F
6482024-02-15T02:57:18.492Z 2 040-059 5 5 5 8 8 8 F F F
6492024-02-15T02:57:18.492Z 3 060-079 5 5 5 8 8 8 F F F
6502024-02-15T02:57:18.492Z 4 080-099 4 4 4 5 5 5 F F F
6512024-02-15T02:57:18.492Z 5 100-119 5 5 5 8 8 8 F F F
6522024-02-15T02:57:18.493Z 6 120-139 3 3 3 3 3 3 F F F
6532024-02-15T02:57:18.493Z 7 140-159 1 1 1 1 1 1 F F F
6542024-02-15T02:57:18.493Z 8 160-179 4 4 4 7 7 7 F F F
6552024-02-15T02:57:18.493Z 9 180-199 5 5 5 8 8 8 F F F
6562024-02-15T02:57:18.493Z 10 200-219 1 1 1 1 1 1 F F F
6572024-02-15T02:57:18.493Z 11 220-239 2 2 2 2 2 2 F F F
6582024-02-15T02:57:18.493Z 12 240-259 1 1 1 1 1 1 F F F
6592024-02-15T02:57:18.493Z 13 260-279 1 1 1 1 1 1 F F F
6602024-02-15T02:57:18.493Z 14 280-299 1 1 1 1 1 1 F F F
6612024-02-15T02:57:18.493Z 15 300-319 4 4 4 6 6 6 F F F
6622024-02-15T02:57:18.493Z 16 320-339 3 3 3 3 3 3 F F F
6632024-02-15T02:57:18.493Z 17 340-359 2 2 2 2 2 2 F F F
6642024-02-15T02:57:18.493Z 18 360-379 5 5 5 8 8 8 F F F
6652024-02-15T02:57:18.493Z 19 380-399 5 5 5 8 8 8 F F F
6662024-02-15T02:57:18.494Z 20 400-419 1 1 1 1 1 1 F F F
6672024-02-15T02:57:18.494Z 21 420-439 3 3 3 3 3 3 F F F
6682024-02-15T02:57:18.494Z 22 440-459 5 5 5 8 8 8 F F F
6692024-02-15T02:57:18.494Z 23 460-479 3 3 3 3 3 3 F F F
6702024-02-15T02:57:18.494Z 24 480-499 1 1 1 1 1 1 F F F
6712024-02-15T02:57:18.494Z 25 500-519 2 2 2 2 2 2 F F F
6722024-02-15T02:57:18.494Z 26 520-539 4 4 4 7 7 7 F F F
6732024-02-15T02:57:18.494Z 27 540-559 4 4 4 7 7 7 F F F
6742024-02-15T02:57:18.494Z 28 560-579 2 2 2 2 2 2 F F F
6752024-02-15T02:57:18.494Z 29 580-599 4 4 4 6 6 6 F F F
6762024-02-15T02:57:18.494ZMax gen: 5, Max flush: 8
6772024-02-15T02:57:18.494Z
6782024-02-15T02:57:18.495ZBegin loop 3
6792024-02-15T02:57:18.495ZDownstairs to restart: 2
6802024-02-15T02:57:18.495Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.444905786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159}
6812024-02-15T02:57:18.495Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"Crucible runtime is spawned
6822024-02-15T02:57:18.495Z82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445279419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159}
6832024-02-15T02:57:18.495Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445312546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159}
6842024-02-15T02:57:18.495Z{"msg":"Crucible d7629600-85d5-4e80-b1d2-73f3eb3aa7b9 has session id: 5f836f7f-98e0-450b-8be7-771698d23037","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445342012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6852024-02-15T02:57:18.495Z{"msg":"Upstairs opts: Upstairs UUID: d7629600-85d5-4e80-b1d2-73f3eb3aa7b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445391496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6862024-02-15T02:57:18.495Z{"msg":"Crucible stats registered with UUID: d7629600-85d5-4e80-b1d2-73f3eb3aa7b9","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445413003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6872024-02-15T02:57:18.495Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445453918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159}
6882024-02-15T02:57:18.495Z{"msg":"d7629600-85d5-4e80-b1d2-73f3eb3aa7b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445481064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6892024-02-15T02:57:18.495Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445508441Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6902024-02-15T02:57:18.495Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445538777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6912024-02-15T02:57:18.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445586731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6922024-02-15T02:57:18.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445613448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6932024-02-15T02:57:18.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445649723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6942024-02-15T02:57:18.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445667871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6952024-02-15T02:57:18.496Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445702586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6962024-02-15T02:57:18.496Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445730973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6972024-02-15T02:57:18.496Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445773038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6982024-02-15T02:57:18.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445798025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
6992024-02-15T02:57:18.496Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:17.445837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7002024-02-15T02:57:18.496Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:17.445881724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7012024-02-15T02:57:18.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.445908201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7022024-02-15T02:57:18.520Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:17.446141771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7032024-02-15T02:57:18.520Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.446187116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7042024-02-15T02:57:18.521Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.446256997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7052024-02-15T02:57:18.521Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.446304631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7062024-02-15T02:57:18.521Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.446519694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7072024-02-15T02:57:18.521Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.446572198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7082024-02-15T02:57:18.521Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.446599924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7092024-02-15T02:57:18.521Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.446624531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7102024-02-15T02:57:18.521Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.447118749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7112024-02-15T02:57:18.521Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.447242534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7122024-02-15T02:57:18.522Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.447645493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7132024-02-15T02:57:18.522Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.447685198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7142024-02-15T02:57:18.522Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.447709005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7152024-02-15T02:57:18.522Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.44774629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7162024-02-15T02:57:18.522Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.447766018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7172024-02-15T02:57:18.522Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:17.447797014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7182024-02-15T02:57:19.598Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.447449291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7192024-02-15T02:57:19.599Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.447543639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7202024-02-15T02:57:19.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.448391713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7212024-02-15T02:57:19.702Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.448439877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7222024-02-15T02:57:19.702Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.448958572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7232024-02-15T02:57:19.702Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.44953402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7242024-02-15T02:57:19.702Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449574504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7252024-02-15T02:57:19.702Z{"msg":"[0]R flush_numbers[0..12]: [5, 6, 8, 8, 5, 8, 3, 1, 7, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449601001Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7262024-02-15T02:57:19.702Z{"msg":"[0]R generation[0..12]: [4, 4, 5, 5, 4, 5, 3, 1, 4, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449633747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7272024-02-15T02:57:19.702Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449660054Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7282024-02-15T02:57:19.702Z{"msg":"[1]R flush_numbers[0..12]: [5, 6, 8, 8, 5, 8, 3, 1, 7, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449684031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7292024-02-15T02:57:19.702Z{"msg":"[1]R generation[0..12]: [4, 4, 5, 5, 4, 5, 3, 1, 4, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449725805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7302024-02-15T02:57:19.702Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
7312024-02-15T02:57:19.702Z,"time":"2024-02-15T02:57:18.44977095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7322024-02-15T02:57:19.702Z{"msg":"[2]R flush_numbers[0..12]: [5, 6, 8, 8, 5, 8, 3, 1, 7, 8, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449804366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7332024-02-15T02:57:19.702Z{"msg":"[2]R generation[0..12]: [4, 4, 5, 5, 4, 5, 3, 1, 4, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.44984497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7342024-02-15T02:57:19.703Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449867018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7352024-02-15T02:57:19.703Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449895284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7362024-02-15T02:57:19.703Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449923351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7372024-02-15T02:57:19.703Z{"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.449970475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7382024-02-15T02:57:19.703Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450025258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7392024-02-15T02:57:19.703Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450044605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7402024-02-15T02:57:19.703Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450069782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7412024-02-15T02:57:19.703Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.45008702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7422024-02-15T02:57:19.703Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450112637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7432024-02-15T02:57:19.703Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450129845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7442024-02-15T02:57:19.703Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450154702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7452024-02-15T02:57:19.703Z{"msg":"d7629600-85d5-4e80-b1d2-73f3eb3aa7b9 is now active with session: 5f836f7f-98e0-450b-8be7-771698d23037","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.45017147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7462024-02-15T02:57:19.703Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450186858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159,"session_id":"5f836f7f-98e0-450b-8be7-771698d23037"}
7472024-02-15T02:57:19.703Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:18.450210485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159}
7482024-02-15T02:57:19.703ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7492024-02-15T02:57:19.703ZLoading write count information from file "/tmp/test_repair_verify.data"
7502024-02-15T02:57:19.703ZRun Repair workload
7512024-02-15T02:57:19.703Z01/30 Write block 232 len 12288 data: 2 2 2
7522024-02-15T02:57:19.703Z02/30 Write block 220 len 20480 data: 2 2 2 2 2
7532024-02-15T02:57:19.703Z03/30 Flush
7542024-02-15T02:57:19.704Z04/30 Write block 276 len 12288 data: 2 2 2
7552024-02-15T02:57:19.704Z05/30 Write block 46 len 20480 data: 2 2 2 2 2
7562024-02-15T02:57:19.704Z06/30 Read block 207 len 28672
7572024-02-15T02:57:19.704Z07/30 Flush
7582024-02-15T02:57:19.704Z08/30 Write block 152 len 8192 data: 2 2
7592024-02-15T02:57:19.704Z09/30 Write block 63 len 4096 data: 3
7602024-02-15T02:57:19.704Z10/30 Write block 259 len 32768 data: 2 2 2 2 2 2 2 2
7612024-02-15T02:57:19.704Z11/30 Read block 300 len 8192
7622024-02-15T02:57:19.704Z12/30 Write block 271 len 32768 data: 2 2 2 2 2 3 3 3
7632024-02-15T02:57:19.704Z13/30 Read block 98 len 24576
7642024-02-15T02:57:19.704Z14/30 Read block 396 len 4096
7652024-02-15T02:57:19.704Z15/30 Flush
7662024-02-15T02:57:19.704Z16/30 Write block 241 len 24576 data: 2 2 2 2 2 2
7672024-02-15T02:57:19.704Z17/30 Read block 190 len 12288
7682024-02-15T02:57:19.705Z18/30 Write block 110 len 32768 data: 3 3 3 3 3 3 3 3
7692024-02-15T02:57:19.705Z19/30 Write block 329 len 28672 data: 2 2 2 2 2 2 3
7702024-02-15T02:57:19.705Z20/30 Read block 122 len 28672
7712024-02-15T02:57:19.705Z21/30 Read block 207 len 12288
7722024-02-15T02:57:19.705Z22/30 Read block 442 len 28672
7732024-02-15T02:57:19.705Z23/30 Write block 192 len 16384 data: 2 2 2 2
7742024-02-15T02:57:19.705Z24/30 Write block 264 len 12288 data: 3 3 3
7752024-02-15T02:57:19.705Z25/30 Read block 13 len 40960
7762024-02-15T02:57:19.705Z26/30 Read block 535 len 28672
7772024-02-15T02:57:19.705Z27/30 Read block 554 len 20480
7782024-02-15T02:57:19.705Z28/30 Write block 573 len 4096 data: 2
7792024-02-15T02:57:19.705Z29/30 Read block 318 len 4096
7802024-02-15T02:57:19.705Z30/30 Write block 4 len 28672 data: 3 3 3 3 3 3 4
7812024-02-15T02:57:19.705Z----------------------------------------------------------------
7822024-02-15T02:57:19.705Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7832024-02-15T02:57:19.705ZStates: Active Active Active
7842024-02-15T02:57:19.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7852024-02-15T02:57:19.705Z 1 Acked 1000 Write 3 Done Done Done false
7862024-02-15T02:57:19.705Z 2 Acked 1001 Write 5 Done Done Sent false
7872024-02-15T02:57:19.705Z 3 Acked 1002 Flush 0 Done Done Sent false
7882024-02-15T02:57:19.705Z 4 Acked 1003 Write 3 Done Done Sent false
7892024-02-15T02:57:19.705Z 5 Acked 1004 Write 5 Done Done Sent false
7902024-02-15T02:57:19.705Z 6 Acked 1005 Read 7 Done Done Sent false
7912024-02-15T02:57:19.705Z 7 Acked 1006 Flush 0 Done Done Sent false
7922024-02-15T02:57:19.706Z 8 Acked 1007 Write 2 Done Done Sent false
7932024-02-15T02:57:19.706Z 9 Acked 1008 Write 1 Done Done Sent false
7942024-02-15T02:57:19.706Z 10 Acked 1009 Write 8 Done Done Sent false
7952024-02-15T02:57:19.706Z 11 Acked 1010 Read 2 Done Done Sent false
7962024-02-15T02:57:19.706Z 12 Acked 1011 Write 8 Done Done Sent false
7972024-02-15T02:57:19.706Z 13 Acked 1012 Read 6 Done Done Sent false
7982024-02-15T02:57:19.706Z 14 Acked 1013 Read 1 Done Done Sent false
7992024-02-15T02:57:19.706Z 15 Acked 1014 Flush 0 Done Done Sent false
8002024-02-15T02:57:19.706Z 16 Acked 1015 Write 6 Done Done Sent false
8012024-02-15T02:57:19.706Z 17 Acked 1016 Read 3 Done Done Sent false
8022024-02-15T02:57:19.706Z 18 Acked 1017 Write 8 Done Done Sent false
8032024-02-15T02:57:19.706Z 19 Acked 1018 Write 7 Done Done Sent false
8042024-02-15T02:57:19.706Z 20 Acked 1019 Read 7 Done Done Sent false
8052024-02-15T02:57:19.706Z 21 Acked 1020 Read 3 Done Done Sent false
8062024-02-15T02:57:19.707Z 22 Acked 1021 Read 7 Done Done Sent false
8072024-02-15T02:57:19.707Z 23 Acked 1022 Write 4 Done Done Sent false
8082024-02-15T02:57:19.707Z 24 Acked 1023 Write 3 Done Done Sent false
8092024-02-15T02:57:19.707Z 25 Acked 1024 Read 10 Done Done Sent false
8102024-02-15T02:57:19.707Z 26 Acked 1025 Read 7 Done Done Sent false
8112024-02-15T02:57:19.707Z 27 Acked 1026 Read 5 Done Done Sent false
8122024-02-15T02:57:19.707Z 28 Acked 1027 Write 1 Done Done Sent false
8132024-02-15T02:57:19.707Z 29 Acked 1028 Read 1 Done Done Sent false
8142024-02-15T02:57:19.707Z 30 Acked 1029 Write 7 Sent Sent Sent false
8152024-02-15T02:57:19.707Z STATES DS:0 DS:1 DS:2 TOTAL
8162024-02-15T02:57:19.707Z New 0 0 0 0
8172024-02-15T02:57:19.707Z Sent 1 1 29 31
8182024-02-15T02:57:19.707Z Done 29 29 1 59
8192024-02-15T02:57:19.707Z Skipped 0 0 0 0
8202024-02-15T02:57:19.819Z Error 0 0 0 0
8212024-02-15T02:57:19.819ZLast Flush: 1014 1014 0
8222024-02-15T02:57:19.819ZDownstairs last five completed:
8232024-02-15T02:57:19.819ZUpstairs last five completed: 30 29 28 27 26
8242024-02-15T02:57:19.819Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:18.479196173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1159}
8252024-02-15T02:57:19.819ZWrote out file "/tmp/test_repair_verify.data"
8262024-02-15T02:57:19.819Z
8272024-02-15T02:57:19.820ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8282024-02-15T02:57:19.820ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8292024-02-15T02:57:19.820Z 0 000-019 4 4 4 5 5 5 T T F <---
8302024-02-15T02:57:19.820Z 1 020-039 4 4 4 6 6 6 F F F
8312024-02-15T02:57:19.820Z 2 040-059 6 6 5 10 10 8 F F F <---
8322024-02-15T02:57:19.820Z 3 060-079 6 6 5 11 11 8 F F F <---
8332024-02-15T02:57:19.820Z 4 080-099 4 4 4 5 5 5 F F F
8342024-02-15T02:57:19.820Z 5 100-119 5 5 5 8 8 8 T T F <---
8352024-02-15T02:57:19.820Z 6 120-139 3 3 3 3 3 3 F F F
8362024-02-15T02:57:19.820Z 7 140-159 6 6 1 11 11 1 F F F <---
8372024-02-15T02:57:19.820Z 8 160-179 4 4 4 7 7 7 F F F
8382024-02-15T02:57:19.820Z 9 180-199 5 5 5 8 8 8 T T F <---
8392024-02-15T02:57:19.820Z 10 200-219 1 1 1 1 1 1 F F F
8402024-02-15T02:57:19.821Z 11 220-239 6 6 2 9 9 2 F F T <---
8412024-02-15T02:57:19.821Z 12 240-259 6 6 1 11 11 1 T T F <---
8422024-02-15T02:57:19.821Z 13 260-279 6 6 1 11 11 1 T T F <---
8432024-02-15T02:57:19.821Z 14 280-299 1 1 1 1 1 1 F F F
8442024-02-15T02:57:19.821Z 15 300-319 4 4 4 6 6 6 F F F
8452024-02-15T02:57:19.821Z 16 320-339 3 3 3 3 3 3 T T F <---
8462024-02-15T02:57:19.821Z 17 340-359 2 2 2 2 2 2 F F F
8472024-02-15T02:57:19.821Z 18 360-379 5 5 5 8 8 8 F F F
8482024-02-15T02:57:19.821Z 19 380-399 5 5 5 8 8 8 F F F
8492024-02-15T02:57:19.821Z 20 400-419 1 1 1 1 1 1 F F F
8502024-02-15T02:57:19.821Z 21 420-439 3 3 3 3 3 3 F F F
8512024-02-15T02:57:19.821Z 22 440-459 5 5 5 8 8 8 F F F
8522024-02-15T02:57:19.821Z 23 460-479 3 3 3 3 3 3 F F F
8532024-02-15T02:57:19.821Z 24 480-499 1 1 1 1 1 1 F F F
8542024-02-15T02:57:19.822Z 25 500-519 2 2 2 2 2 2 F F F
8552024-02-15T02:57:19.822Z 26 520-539 4 4 4 7 7 7 F F F
8562024-02-15T02:57:19.822Z 27 540-559 4 4 4 7 7 7 F F F
8572024-02-15T02:57:19.822Z 28 560-579 2 2 2 2 2 2 T T F <---
8582024-02-15T02:57:19.822Z 29 580-599 4 4 4 6 6 6 F F F
8592024-02-15T02:57:19.822ZMax gen: 6, Max flush: 11
8602024-02-15T02:57:19.822ZError: Difference in extent metadata found!
8612024-02-15T02:57:19.822ZA Difference in extent metadata is expected here
8622024-02-15T02:57:19.822ZOn loop 3
8632024-02-15T02:57:19.822Z
8642024-02-15T02:57:19.822ZVerifying data now
8652024-02-15T02:57:33.033ZNo change after verify
8662024-02-15T02:57:33.033ZLoop: 3 Downstairs dump after verify (and repair):
8672024-02-15T02:57:33.033ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8682024-02-15T02:57:33.033Z 0 000-019 7 7 7 12 12 12 F F F
8692024-02-15T02:57:33.752Z 1 020-039 4 4 4 6 6 6 F F F
8702024-02-15T02:57:33.752Z 2 040-059 6 6 6 10 10 10 F F F
8712024-02-15T02:57:33.753Z 3 060-079 6 6 6 11 11 11 F F F
8722024-02-15T02:57:33.753Z 4 080-099 4 4 4 5 5 5 F F F
8732024-02-15T02:57:33.753Z 5 100-119 7 7 7 12 12 12 F F F
8742024-02-15T02:57:33.753Z 6 120-139 3 3 3 3 3 3 F F F
8752024-02-15T02:57:33.753Z 7 140-159 6 6 6 11 11 11 F F F
8762024-02-15T02:57:33.753Z 8 160-179 4 4 4 7 7 7 F F F
8772024-02-15T02:57:33.753Z 9 180-199 7 7 7 12 12 12 F F F
8782024-02-15T02:57:33.753Z 10 200-219 1 1 1 1 1 1 F F F
8792024-02-15T02:57:33.753Z 11 220-239 6 6 6 9 9 9 F F F
8802024-02-15T02:57:33.753Z 12 240-259 7 7 7 12 12 12 F F F
8812024-02-15T02:57:33.753Z 13 260-279 7 7 7 12 12 12 F F F
8822024-02-15T02:57:33.753Z 14 280-299 1 1 1 1 1 1 F F F
8832024-02-15T02:57:33.753Z 15 300-319 4 4 4 6 6 6 F F F
8842024-02-15T02:57:33.754Z 16 320-339 7 7 7 12 12 12 F F F
8852024-02-15T02:57:33.754Z 17 340-359 2 2 2 2 2 2 F F F
8862024-02-15T02:57:33.754Z 18 360-379 5 5 5 8 8 8 F F F
8872024-02-15T02:57:33.754Z 19 380-399 5 5 5 8 8 8 F F F
8882024-02-15T02:57:33.754Z 20 400-419 1 1 1 1 1 1 F F F
8892024-02-15T02:57:33.754Z 21 420-439 3 3 3 3 3 3 F F F
8902024-02-15T02:57:33.754Z 22 440-459 5 5 5 8 8 8 F F F
8912024-02-15T02:57:33.754Z 23 460-479 3 3 3 3 3 3 F F F
8922024-02-15T02:57:33.754Z 24 480-499 1 1 1 1 1 1 F F F
8932024-02-15T02:57:33.754Z 25 500-519 2 2 2 2 2 2 F F F
8942024-02-15T02:57:33.754Z 26 520-539 4 4 4 7 7 7 F F F
8952024-02-15T02:57:33.754Z 27 540-559 4 4 4 7 7 7 F F F
8962024-02-15T02:57:33.754Z 28 560-579 7 7 7 12 12 12 F F F
8972024-02-15T02:57:33.754Z 29 580-599 4 4 4 6 6 6 F F F
8982024-02-15T02:57:33.755ZMax gen: 7, Max flush: 12
8992024-02-15T02:57:33.755Z
9002024-02-15T02:57:33.755ZBegin loop 4
9012024-02-15T02:57:33.755ZDownstairs to restart: 1
9022024-02-15T02:57:33.755Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.120589873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167}
9032024-02-15T02:57:33.755Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
9042024-02-15T02:57:33.755Z,"time":"2024-02-15T02:57:24.120940549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167}
9052024-02-15T02:57:33.755Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.120967795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167}
9062024-02-15T02:57:33.755Z{"msg":"Crucible aafa1def-bae0-44fc-b99e-93cf61700157 has session id: d83431db-6f96-4916-84aa-a78289e9b17a","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.12101303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9072024-02-15T02:57:33.755Z{"msg":"Upstairs opts: Upstairs UUID: aafa1def-bae0-44fc-b99e-93cf61700157, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121059744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9082024-02-15T02:57:33.755Z{"msg":"Crucible stats registered with UUID: aafa1def-bae0-44fc-b99e-93cf61700157","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.12109153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9092024-02-15T02:57:33.755Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121122196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167}
9102024-02-15T02:57:33.755Z{"msg":"aafa1def-bae0-44fc-b99e-93cf61700157 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121141854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9112024-02-15T02:57:33.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.12117322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9122024-02-15T02:57:33.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121206745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9132024-02-15T02:57:33.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.12124971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9142024-02-15T02:57:33.756Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121283306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9152024-02-15T02:57:33.756Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121304643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9162024-02-15T02:57:34.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121326941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9172024-02-15T02:57:34.134Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121358617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9182024-02-15T02:57:34.134Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121398712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9192024-02-15T02:57:34.134Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121425048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9202024-02-15T02:57:34.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121553722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9212024-02-15T02:57:34.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121614534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9222024-02-15T02:57:34.135Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:24.121659279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9232024-02-15T02:57:34.135Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:24.121697894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9242024-02-15T02:57:34.135Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:24.121721261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9252024-02-15T02:57:34.135Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121754877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9262024-02-15T02:57:34.135Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121793512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9272024-02-15T02:57:34.135Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.121899809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9282024-02-15T02:57:34.136Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.122144308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9292024-02-15T02:57:34.136Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.122181523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9302024-02-15T02:57:34.136Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.122356032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9312024-02-15T02:57:34.136Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.122398566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9322024-02-15T02:57:34.136Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.12276617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9332024-02-15T02:57:34.136Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.12292781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9342024-02-15T02:57:34.136Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.123295664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9352024-02-15T02:57:34.136Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.123334379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9362024-02-15T02:57:34.136Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.123356526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9372024-02-15T02:57:34.136Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.123370314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9382024-02-15T02:57:34.136Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.123384912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9392024-02-15T02:57:34.136Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:24.123411739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9402024-02-15T02:57:34.137Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.123374877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9412024-02-15T02:57:34.137Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.123531428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9422024-02-15T02:57:34.137Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.124403718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9432024-02-15T02:57:34.137Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.124439534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9442024-02-15T02:57:34.137Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.1249456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9452024-02-15T02:57:34.137Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125531657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9462024-02-15T02:57:34.137Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.12558481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9472024-02-15T02:57:34.137Z{"msg":"[0]R flush_numbers[0..12]: [12, 6, 10, 11, 5, 12, 3, 11, 7, 12, 1, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125623385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9482024-02-15T02:57:34.137Z{"msg":"[0]R generation[0..12]: [7, 4, 6, 6, 4, 7, 3, 6, 4, 7, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125651072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9492024-02-15T02:57:34.137Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125674099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9502024-02-15T02:57:34.137Z{"msg":"[1]R flush_numbers[0..12]: [12, 6, 10, 11, 5, 12, 3, 11, 7, 12, 1, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125690047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9512024-02-15T02:57:34.138Z{"msg":"[1]R generation[0..12]: [7, 4, 6, 6, 4, 7, 3, 6, 4, 7, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125710874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9522024-02-15T02:57:34.138Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125726053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9532024-02-15T02:57:34.138Z{"msg":"[2]R flush_numbers[0..12]: [12, 6, 10, 11, 5, 12, 3, 11, 7, 12, 1, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125740571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9542024-02-15T02:57:34.138Z{"msg":"[2]R generation[0..12]: [7, 4, 6, 6, 4, 7, 3, 6, 4, 7, 1, 6]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
9552024-02-15T02:57:34.138Z,"time":"2024-02-15T02:57:25.125761328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9562024-02-15T02:57:34.138Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125780616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9572024-02-15T02:57:34.138Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125800873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9582024-02-15T02:57:34.138Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125815511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9592024-02-15T02:57:34.139Z{"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125835089Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9602024-02-15T02:57:34.139Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125849247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9612024-02-15T02:57:34.139Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125869904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9622024-02-15T02:57:34.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125889432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9632024-02-15T02:57:34.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.12590305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9642024-02-15T02:57:34.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125922778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9652024-02-15T02:57:34.139Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125937476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9662024-02-15T02:57:34.139ZError: Difference in extent metadata found!
9672024-02-15T02:57:34.139Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125956513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9682024-02-15T02:57:34.139Z{"msg":"aafa1def-bae0-44fc-b99e-93cf61700157 is now active with session: d83431db-6f96-4916-84aa-a78289e9b17a","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125969592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9692024-02-15T02:57:34.140Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.125981441Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167,"session_id":"d83431db-6f96-4916-84aa-a78289e9b17a"}
9702024-02-15T02:57:34.140Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:25.126007067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167}
9712024-02-15T02:57:34.140ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9722024-02-15T02:57:34.140ZLoading write count information from file "/tmp/test_repair_verify.data"
9732024-02-15T02:57:34.140ZRun Repair workload
9742024-02-15T02:57:34.140Z01/30 Write block 16 len 20480 data: 2 2 2 2 2
9752024-02-15T02:57:34.140Z02/30 Flush
9762024-02-15T02:57:34.140Z03/30 Write block 333 len 28672 data: 3 3 4 3 3 3 3
9772024-02-15T02:57:34.140Z04/30 Read block 585 len 24576
9782024-02-15T02:57:34.140Z05/30 Write block 44 len 28672 data: 2 2 3 3 3 3 3
9792024-02-15T02:57:34.141Z06/30 Read block 405 len 28672
9802024-02-15T02:57:34.141Z07/30 Flush
9812024-02-15T02:57:34.141Z08/30 Write block 309 len 24576 data: 4 4 3 3 2 2
9822024-02-15T02:57:34.141Z09/30 Write block 491 len 32768 data: 2 2 2 2 2 2 2 2
9832024-02-15T02:57:34.141Z10/30 Write block 439 len 40960 data: 2 2 2 3 3 3 3 3 3 2
9842024-02-15T02:57:34.141Z11/30 Read block 545 len 8192
9852024-02-15T02:57:34.141Z12/30 Write block 347 len 28672 data: 2 2 2 2 2 2 2
9862024-02-15T02:57:34.141Z13/30 Write block 558 len 20480 data: 3 3 3 3 3
9872024-02-15T02:57:34.141Z14/30 Read block 516 len 8192
9882024-02-15T02:57:34.142Z15/30 Read block 48 len 4096
9892024-02-15T02:57:34.142Z16/30 Write block 38 len 4096 data: 2
9902024-02-15T02:57:34.142Z17/30 Write block 487 len 20480 data: 2 2 2 2 3
9912024-02-15T02:57:34.142Z18/30 Read block 216 len 8192
9922024-02-15T02:57:34.142Z19/30 Write block 330 len 8192 data: 3 3
9932024-02-15T02:57:34.142Z20/30 Write block 304 len 40960 data: 4 5 5 5 4 5 5 4 4 3
9942024-02-15T02:57:34.142Z21/30 Write block 469 len 24576 data: 3 3 3 2 2 2
9952024-02-15T02:57:34.143Z22/30 Write block 265 len 36864 data: 4 4 2 2 2 2 3 3 3
9962024-02-15T02:57:34.143Z23/30 Read block 474 len 16384
9972024-02-15T02:57:34.143Z24/30 Read block 574 len 4096
9982024-02-15T02:57:34.143Z25/30 Write block 383 len 8192 data: 3 3
9992024-02-15T02:57:34.143Z26/30 Write block 509 len 40960 data: 2 2 2 3 3 3 3 3 2 2
10002024-02-15T02:57:34.144Z27/30 Read block 506 len 36864
10012024-02-15T02:57:34.144Z28/30 Read block 452 len 20480
10022024-02-15T02:57:34.144Z29/30 Read block 221 len 20480
10032024-02-15T02:57:34.144Z30/30 Read block 276 len 32768
10042024-02-15T02:57:34.144Z----------------------------------------------------------------
10052024-02-15T02:57:34.144Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
10062024-02-15T02:57:34.144ZStates: Active Active Active
10072024-02-15T02:57:34.144ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
10082024-02-15T02:57:34.145Z 1 Acked 1000 Write 5 Done Done Done false
10092024-02-15T02:57:34.145Z 2 Acked 1001 Flush 0 Done Sent Done false
10102024-02-15T02:57:34.145Z 3 Acked 1002 Write 7 Done Sent Done false
10112024-02-15T02:57:34.145Z 4 Acked 1003 Read 6 Done Sent Done false
10122024-02-15T02:57:34.145Z 5 Acked 1004 Write 7 Done Sent Done false
10132024-02-15T02:57:34.145Z 6 Acked 1005 Read 7 Done Sent Done false
10142024-02-15T02:57:34.145Z 7 Acked 1006 Flush 0 Done Sent Done false
10152024-02-15T02:57:34.145Z 8 Acked 1007 Write 6 Done Sent Done false
10162024-02-15T02:57:34.388Z 9 Acked 1008 Write 8 Done Sent Done false
10172024-02-15T02:57:34.388Z 10 Acked 1009 Write 10 Done Sent Done false
10182024-02-15T02:57:34.388Z 11 Acked 1010 Read 2 Done Sent Done false
10192024-02-15T02:57:34.389Z 12 Acked 1011 Write 7 Done Sent Done false
10202024-02-15T02:57:34.389Z 13 Acked 1012 Write 5 Done Sent Done false
10212024-02-15T02:57:34.389Z 14 Acked 1013 Read 2 Done Sent Done false
10222024-02-15T02:57:34.389Z 15 Acked 1014 Read 1 Done Sent Done false
10232024-02-15T02:57:34.389Z 16 Acked 1015 Write 1 Done Sent Done false
10242024-02-15T02:57:34.389Z 17 Acked 1016 Write 5 Done Sent Done false
10252024-02-15T02:57:34.390Z 18 Acked 1017 Read 2 Done Sent Done false
10262024-02-15T02:57:34.390Z 19 Acked 1018 Write 2 Done Sent Done false
10272024-02-15T02:57:34.390Z 20 Acked 1019 Write 10 Done Sent Done false
10282024-02-15T02:57:34.390Z 21 Acked 1020 Write 6 Done Sent Done false
10292024-02-15T02:57:34.390Z 22 Acked 1021 Write 9 Done Sent Done false
10302024-02-15T02:57:34.390Z 23 Acked 1022 Read 4 Done Sent Done false
10312024-02-15T02:57:34.390Z 24 Acked 1023 Read 1 Done Sent Done false
10322024-02-15T02:57:34.391Z 25 Acked 1024 Write 2 Done Sent Done false
10332024-02-15T02:57:34.391Z 26 Acked 1025 Write 10 Done Sent Done false
10342024-02-15T02:57:34.391Z 27 Acked 1026 Read 9 Done Sent Done false
10352024-02-15T02:57:34.391Z 28 Acked 1027 Read 5 Done Sent Done false
10362024-02-15T02:57:34.391Z 29 Acked 1028 Read 5 Done Sent Done false
10372024-02-15T02:57:34.391Z 30 Acked 1029 Read 8 Done Sent Done false
10382024-02-15T02:57:34.391Z STATES DS:0 DS:1 DS:2 TOTAL
10392024-02-15T02:57:34.391Z New 0 0 0 0
10402024-02-15T02:57:34.391Z Sent 0 29 0 29
10412024-02-15T02:57:34.391Z Done 30 1 30 61
10422024-02-15T02:57:34.392Z Skipped 0 0 0 0
10432024-02-15T02:57:34.392Z Error 0 0 0 0
10442024-02-15T02:57:34.392ZLast Flush: 1006 0 1006
10452024-02-15T02:57:34.392ZDownstairs last five completed:
10462024-02-15T02:57:34.392ZUpstairs last five completed: 30 29 28 27 26
10472024-02-15T02:57:34.392Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:25.156627551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1167}
10482024-02-15T02:57:34.392ZWrote out file "/tmp/test_repair_verify.data"
10492024-02-15T02:57:34.392Z
10502024-02-15T02:57:34.392ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10512024-02-15T02:57:34.392ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10522024-02-15T02:57:34.392Z 0 000-019 8 7 8 13 12 13 F T F <---
10532024-02-15T02:57:34.392Z 1 020-039 8 4 8 13 6 13 T T T <---
10542024-02-15T02:57:34.392Z 2 040-059 8 6 8 14 10 14 F F F <---
10552024-02-15T02:57:34.392Z 3 060-079 6 6 6 11 11 11 F F F
10562024-02-15T02:57:34.393Z 4 080-099 4 4 4 5 5 5 F F F
10572024-02-15T02:57:34.393Z 5 100-119 7 7 7 12 12 12 F F F
10582024-02-15T02:57:34.393Z 6 120-139 3 3 3 3 3 3 F F F
10592024-02-15T02:57:34.393Z 7 140-159 6 6 6 11 11 11 F F F
10602024-02-15T02:57:34.393Z 8 160-179 4 4 4 7 7 7 F F F
10612024-02-15T02:57:34.393Z 9 180-199 7 7 7 12 12 12 F F F
10622024-02-15T02:57:34.393Z 10 200-219 1 1 1 1 1 1 F F F
10632024-02-15T02:57:34.393Z 11 220-239 6 6 6 9 9 9 F F F
10642024-02-15T02:57:34.393Z 12 240-259 7 7 7 12 12 12 F F F
10652024-02-15T02:57:34.393Z 13 260-279 7 7 7 12 12 12 T F T <---
10662024-02-15T02:57:34.393Z 14 280-299 1 1 1 1 1 1 F F F
10672024-02-15T02:57:34.393Z 15 300-319 4 4 4 6 6 6 T F T <---
10682024-02-15T02:57:34.393Z 16 320-339 8 7 8 14 12 14 T F T <---
10692024-02-15T02:57:34.393Z 17 340-359 2 2 2 2 2 2 T F T <---
10702024-02-15T02:57:34.394Z 18 360-379 5 5 5 8 8 8 F F F
10712024-02-15T02:57:34.394Z 19 380-399 5 5 5 8 8 8 T F T <---
10722024-02-15T02:57:34.394Z 20 400-419 1 1 1 1 1 1 F F F
10732024-02-15T02:57:34.394Z 21 420-439 3 3 3 3 3 3 T F T <---
10742024-02-15T02:57:34.394Z 22 440-459 5 5 5 8 8 8 T F T <---
10752024-02-15T02:57:34.394Z 23 460-479 3 3 3 3 3 3 T F T <---
10762024-02-15T02:57:34.394Z 24 480-499 1 1 1 1 1 1 T F T <---
10772024-02-15T02:57:34.394Z 25 500-519 2 2 2 2 2 2 T F T <---
10782024-02-15T02:57:34.394Z 26 520-539 4 4 4 7 7 7 F F F
10792024-02-15T02:57:34.394Z 27 540-559 4 4 4 7 7 7 T F T <---
10802024-02-15T02:57:34.394Z 28 560-579 7 7 7 12 12 12 T F T <---
10812024-02-15T02:57:34.394Z 29 580-599 4 4 4 6 6 6 F F F
10822024-02-15T02:57:34.394ZMax gen: 8, Max flush: 14
10832024-02-15T02:57:34.395ZA Difference in extent metadata is expected here
10842024-02-15T02:57:34.395ZOn loop 4
10852024-02-15T02:57:34.395Z
10862024-02-15T02:57:34.395ZVerifying data now
10872024-02-15T02:57:34.395ZNo change after verify
10882024-02-15T02:57:34.395ZLoop: 4 Downstairs dump after verify (and repair):
10892024-02-15T02:57:34.395ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10902024-02-15T02:57:34.395Z 0 000-019 8 8 8 13 13 13 F F F
10912024-02-15T02:57:34.395Z 1 020-039 9 9 9 15 15 15 F F F
10922024-02-15T02:57:34.395Z 2 040-059 8 8 8 14 14 14 F F F
10932024-02-15T02:57:34.396Z 3 060-079 6 6 6 11 11 11 F F F
10942024-02-15T02:57:34.396Z 4 080-099 4 4 4 5 5 5 F F F
10952024-02-15T02:57:34.396Z 5 100-119 7 7 7 12 12 12 F F F
10962024-02-15T02:57:34.396Z 6 120-139 3 3 3 3 3 3 F F F
10972024-02-15T02:57:34.396Z 7 140-159 6 6 6 11 11 11 F F F
10982024-02-15T02:57:34.396Z 8 160-179 4 4 4 7 7 7 F F F
10992024-02-15T02:57:34.396Z 9 180-199 7 7 7 12 12 12 F F F
11002024-02-15T02:57:34.396Z 10 200-219 1 1 1 1 1 1 F F F
11012024-02-15T02:57:34.397Z 11 220-239 6 6 6 9 9 9 F F F
11022024-02-15T02:57:34.397Z 12 240-259 7 7 7 12 12 12 F F F
11032024-02-15T02:57:34.397Z 13 260-279 9 9 9 15 15 15 F F F
11042024-02-15T02:57:34.397Z 14 280-299 1 1 1 1 1 1 F F F
11052024-02-15T02:57:34.397Z 15 300-319 9 9 9 15 15 15 F F F
11062024-02-15T02:57:34.397Z 16 320-339 9 9 9 15 15 15 F F F
11072024-02-15T02:57:34.398Z 17 340-359 9 9 9 15 15 15 F F F
11082024-02-15T02:57:34.398Z 18 360-379 5 5 5 8 8 8 F F F
11092024-02-15T02:57:34.398Z 19 380-399 9 9 9 15 15 15 F F F
11102024-02-15T02:57:34.398Z 20 400-419 1 1 1 1 1 1 F F F
11112024-02-15T02:57:34.398Z 21 420-439 9 9 9 15 15 15 F F F
11122024-02-15T02:57:34.398Z 22 440-459 9 9 9 15 15 15 F F F
11132024-02-15T02:57:34.398Z 23 460-479 9 9 9 15 15 15 F F F
11142024-02-15T02:57:34.398Z 24 480-499 9 9 9 15 15 15 F F F
11152024-02-15T02:57:34.398Z 25 500-519 9 9 9 15 15 15 F F F
11162024-02-15T02:57:34.595Z 26 520-539 4 4 4 7 7 7 F F F
11172024-02-15T02:57:34.595Z 27 540-559 9 9 9 15 15 15 F F F
11182024-02-15T02:57:34.595Z 28 560-579 9 9 9 15 15 15 F F F
11192024-02-15T02:57:34.595Z 29 580-599 4 4 4 6 6 6 F F F
11202024-02-15T02:57:34.595ZMax gen: 9, Max flush: 15
11212024-02-15T02:57:34.596Z
11222024-02-15T02:57:34.596ZBegin loop 5
11232024-02-15T02:57:34.596ZDownstairs to restart: 2
11242024-02-15T02:57:34.596Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970062435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175}
11252024-02-15T02:57:34.596Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"Crucible runtime is spawned
11262024-02-15T02:57:34.596Zx86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970452056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175}
11272024-02-15T02:57:34.596Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970492061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175}
11282024-02-15T02:57:34.597Z{"msg":"Crucible 6a2ca697-8569-4f28-b3cc-0b3025bcfcc9 has session id: 58542f12-ae3d-40c2-b967-5eda0894435d","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970521677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11292024-02-15T02:57:34.597Z{"msg":"Upstairs opts: Upstairs UUID: 6a2ca697-8569-4f28-b3cc-0b3025bcfcc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970571211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11302024-02-15T02:57:34.597Z{"msg":"Crucible stats registered with UUID: 6a2ca697-8569-4f28-b3cc-0b3025bcfcc9","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970603897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11312024-02-15T02:57:34.597Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970631343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175}
11322024-02-15T02:57:34.597Z{"msg":"6a2ca697-8569-4f28-b3cc-0b3025bcfcc9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970647061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11332024-02-15T02:57:34.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970663119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11342024-02-15T02:57:34.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970681587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11352024-02-15T02:57:34.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970698625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11362024-02-15T02:57:34.598Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970715633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11372024-02-15T02:57:34.598Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.9707327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11382024-02-15T02:57:34.598Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970755098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11392024-02-15T02:57:34.598Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970772586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11402024-02-15T02:57:34.598Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970796113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11412024-02-15T02:57:34.598Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970826199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11422024-02-15T02:57:34.599Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.970867894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11432024-02-15T02:57:34.599Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:30.970906509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11442024-02-15T02:57:34.599Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:30.970942724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11452024-02-15T02:57:34.599Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.97097467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11462024-02-15T02:57:34.599Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:30.971010636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11472024-02-15T02:57:34.599Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.971042462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11482024-02-15T02:57:34.599Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.971064199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11492024-02-15T02:57:34.600Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.971211591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11502024-02-15T02:57:34.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.971667163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11512024-02-15T02:57:34.600Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:30.97169682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11522024-02-15T02:57:34.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358639747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11532024-02-15T02:57:34.601Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358728886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11542024-02-15T02:57:34.601Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358769921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11552024-02-15T02:57:34.601Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358805726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11562024-02-15T02:57:34.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358839252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11572024-02-15T02:57:34.602Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358873108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11582024-02-15T02:57:34.602Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358906104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11592024-02-15T02:57:34.603Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.35893613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11602024-02-15T02:57:34.603Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.358981654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11612024-02-15T02:57:34.603Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.359032448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11622024-02-15T02:57:34.603Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.972508341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11632024-02-15T02:57:34.603Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.972580612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11642024-02-15T02:57:34.604Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.973497547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11652024-02-15T02:57:34.604Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.973522464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11662024-02-15T02:57:34.604Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974058537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11672024-02-15T02:57:34.604Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974661471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11682024-02-15T02:57:34.605Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974692007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11692024-02-15T02:57:34.605Z{"msg":"[0]R flush_numbers[0..12]: [13, 15, 14, 11, 5, 12, 3, 11, 7, 12, 1, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974710555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11702024-02-15T02:57:34.605Z{"msg":"[0]R generation[0..12]: [8, 9, 8, 6, 4, 7, 3, 6, 4, 7, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974728542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11712024-02-15T02:57:34.605Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.97474705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11722024-02-15T02:57:34.605Z{"msg":"[1]R flush_numbers[0..12]: [13, 15, 14, 11, 5, 12, 3, 11, 7, 12, 1, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974765548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11732024-02-15T02:57:34.605Z{"msg":"[1]R generation[0..12]: [8, 9, 8, 6, 4, 7, 3, 6, 4, 7, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974790605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11742024-02-15T02:57:34.606Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974809092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11752024-02-15T02:57:34.606Z{"msg":"[2]R flush_numbers[0..12]: [13, 15, 14, 11, 5, 12, 3, 11, 7, 12, 1, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.97482681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11762024-02-15T02:57:34.606Z{"msg":"[2]R generation[0..12]: [8, 9, 8, 6, 4, 7, 3, 6, 4, 7, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974844128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11772024-02-15T02:57:34.606Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
11782024-02-15T02:57:34.607Z,"time":"2024-02-15T02:57:31.974862066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11792024-02-15T02:57:34.607Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974879274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11802024-02-15T02:57:34.607Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:31.974896692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11812024-02-15T02:57:34.607Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11822024-02-15T02:57:34.607ZNext flush: 16","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168089585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11832024-02-15T02:57:34.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168135299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11842024-02-15T02:57:34.607Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168149488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11852024-02-15T02:57:34.608Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168162176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11862024-02-15T02:57:34.608Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168184763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11872024-02-15T02:57:34.608Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168198752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11882024-02-15T02:57:34.608Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168219639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11892024-02-15T02:57:34.608Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168233737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11902024-02-15T02:57:34.608Z{"msg":"6a2ca697-8569-4f28-b3cc-0b3025bcfcc9 is now active with session: 58542f12-ae3d-40c2-b967-5eda0894435d","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168253974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11912024-02-15T02:57:34.608Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168266713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175,"session_id":"58542f12-ae3d-40c2-b967-5eda0894435d"}
11922024-02-15T02:57:34.609Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:32.168285901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175}
11932024-02-15T02:57:34.609ZLoading write count information from file "/tmp/test_repair_verify.data"
11942024-02-15T02:57:34.609ZRun Repair workload
11952024-02-15T02:57:34.609Z01/30 Write block 214 len 40960 data: 2 2 2 2 2 2 3 3 3 3
11962024-02-15T02:57:34.609Z02/30 Read block 18 len 8192
11972024-02-15T02:57:34.609Z03/30 Write block 242 len 36864 data: 3 3 3 3 3 2 2 2 2
11982024-02-15T02:57:34.610Z04/30 Flush
11992024-02-15T02:57:34.610Z05/30 Flush
12002024-02-15T02:57:34.610Z06/30 Write block 300 len 4096 data: 2
12012024-02-15T02:57:34.610Z07/30 Read block 61 len 4096
12022024-02-15T02:57:34.610Z08/30 Read block 582 len 4096
12032024-02-15T02:57:34.610Z09/30 Flush
12042024-02-15T02:57:34.610Z10/30 Write block 486 len 8192 data: 2 3
12052024-02-15T02:57:34.611Z11/30 Write block 542 len 8192 data: 3 2
12062024-02-15T02:57:34.611Z12/30 Read block 578 len 12288
12072024-02-15T02:57:34.611Z13/30 Write block 211 len 16384 data: 2 2 2 3
12082024-02-15T02:57:34.611Z14/30 Read block 38 len 12288
12092024-02-15T02:57:34.611Z15/30 Read block 547 len 28672
12102024-02-15T02:57:34.611Z16/30 Write block 68 len 16384 data: 2 2 2 2
12112024-02-15T02:57:34.611Z17/30 Read block 462 len 40960
12122024-02-15T02:57:34.612Z18/30 Read block 541 len 8192
12132024-02-15T02:57:34.612Z19/30 Read block 90 len 40960
12142024-02-15T02:57:34.612Z20/30 Read block 273 len 16384
12152024-02-15T02:57:34.612Z21/30 Write block 559 len 28672 data: 4 4 4 4 3 3 3
12162024-02-15T02:57:45.940Z22/30 Read block 255 len 32768
12172024-02-15T02:57:45.940Z23/30 Write block 407 len 32768 data: 2 2 2 2 2 2 2 2
12182024-02-15T02:57:45.940Z24/30 Write block 193 len 36864 data: 3 3 3 2 2 2 2 2 2
12192024-02-15T02:57:45.940Z25/30 Read block 463 len 36864
12202024-02-15T02:57:45.941Z26/30 Read block 19 len 4096
12212024-02-15T02:57:45.941Z27/30 Read block 162 len 32768
12222024-02-15T02:57:45.941Z28/30 Write block 35 len 4096 data: 3
12232024-02-15T02:57:45.941Z29/30 Read block 542 len 36864
12242024-02-15T02:57:45.941Z30/30 Write block 219 len 16384 data: 3 4 4 4
12252024-02-15T02:57:45.942Z----------------------------------------------------------------
12262024-02-15T02:57:45.942Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:21
12272024-02-15T02:57:45.942ZStates: Active Active Active
12282024-02-15T02:57:45.942ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
12292024-02-15T02:57:45.942Z 10 Acked 1009 Write 2 Done Done Sent false
12302024-02-15T02:57:45.942Z 11 Acked 1010 Write 2 Done Done Sent false
12312024-02-15T02:57:45.942Z 12 Acked 1011 Read 3 Done Done Sent false
12322024-02-15T02:57:45.942Z 13 Acked 1012 Write 4 Done Done Sent false
12332024-02-15T02:57:45.942Z 14 Acked 1013 Read 3 Done Done Sent false
12342024-02-15T02:57:45.943Z 15 Acked 1014 Read 7 Done Done Sent false
12352024-02-15T02:57:45.943Z 16 Acked 1015 Write 4 Done Done Sent false
12362024-02-15T02:57:45.943Z 17 Acked 1016 Read 10 Done Done Sent false
12372024-02-15T02:57:45.943Z 18 Acked 1017 Read 2 Done Done Sent false
12382024-02-15T02:57:45.943Z 19 Acked 1018 Read 10 Done Done Sent false
12392024-02-15T02:57:45.943Z 20 Acked 1019 Read 4 Done Done Sent false
12402024-02-15T02:57:45.943Z 21 Acked 1020 Write 7 Done Done Sent false
12412024-02-15T02:57:45.944Z 22 Acked 1021 Read 8 Done Done Sent false
12422024-02-15T02:57:45.944Z 23 Acked 1022 Write 8 Done Done Sent false
12432024-02-15T02:57:45.944Z 24 Acked 1023 Write 9 Done Done Sent false
12442024-02-15T02:57:45.944Z 25 Acked 1024 Read 9 Done Done Sent false
12452024-02-15T02:57:45.944Z 26 Acked 1025 Read 1 Done Done Sent false
12462024-02-15T02:57:45.944Z 27 Acked 1026 Read 8 Done Done Sent false
12472024-02-15T02:57:45.944Z 28 Acked 1027 Write 1 Done Done Sent false
12482024-02-15T02:57:45.945Z 29 Acked 1028 Read 9 Done Done Sent false
12492024-02-15T02:57:45.945Z 30 Acked 1029 Write 4 Sent Sent Sent false
12502024-02-15T02:57:45.945Z STATES DS:0 DS:1 DS:2 TOTAL
12512024-02-15T02:57:45.945Z New 0 0 0 0
12522024-02-15T02:57:45.945Z Sent 1 1 21 23
12532024-02-15T02:57:45.945Z Done 20 20 0 40
12542024-02-15T02:57:45.945Z Skipped 0 0 0 0
12552024-02-15T02:57:45.946Z Error 0 0 0 0
12562024-02-15T02:57:45.946ZLast Flush: 1008 1008 1008
12572024-02-15T02:57:45.946ZDownstairs last five completed: 1008 1007 1006 1005 1004
12582024-02-15T02:57:45.947ZUpstairs last five completed: 30 29 28 27 26
12592024-02-15T02:57:45.947Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:32.197511169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1175}
12602024-02-15T02:57:45.947ZWrote out file "/tmp/test_repair_verify.data"
12612024-02-15T02:57:45.947Z
12622024-02-15T02:57:45.947ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12632024-02-15T02:57:45.947ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12642024-02-15T02:57:45.947Z 0 000-019 8 8 8 13 13 13 F F F
12652024-02-15T02:57:45.947Z 1 020-039 9 9 9 15 15 15 T T F <---
12662024-02-15T02:57:45.948Z 2 040-059 8 8 8 14 14 14 F F F
12672024-02-15T02:57:45.948Z 3 060-079 6 6 6 11 11 11 T T F <---
12682024-02-15T02:57:45.948Z 4 080-099 4 4 4 5 5 5 F F F
12692024-02-15T02:57:45.948Z 5 100-119 7 7 7 12 12 12 F F F
12702024-02-15T02:57:45.948Z 6 120-139 3 3 3 3 3 3 F F F
12712024-02-15T02:57:45.948Z 7 140-159 6 6 6 11 11 11 F F F
12722024-02-15T02:57:45.948Z 8 160-179 4 4 4 7 7 7 F F F
12732024-02-15T02:57:45.948Z 9 180-199 7 7 7 12 12 12 T T F <---
12742024-02-15T02:57:45.948ZError: Difference in extent metadata found!
12752024-02-15T02:57:45.948Z 10 200-219 10 10 10 16 16 16 T T F <---
12762024-02-15T02:57:45.948Z 11 220-239 10 10 10 16 16 16 T T F <---
12772024-02-15T02:57:45.948Z 12 240-259 10 10 10 16 16 16 F F F
12782024-02-15T02:57:45.948Z 13 260-279 9 9 9 15 15 15 F F F
12792024-02-15T02:57:45.948Z 14 280-299 1 1 1 1 1 1 F F F
12802024-02-15T02:57:45.949Z 15 300-319 10 10 10 18 18 18 F F F
12812024-02-15T02:57:45.949Z 16 320-339 9 9 9 15 15 15 F F F
12822024-02-15T02:57:45.949Z 17 340-359 9 9 9 15 15 15 F F F
12832024-02-15T02:57:45.949Z 18 360-379 5 5 5 8 8 8 F F F
12842024-02-15T02:57:45.949Z 19 380-399 9 9 9 15 15 15 F F F
12852024-02-15T02:57:45.949Z 20 400-419 1 1 1 1 1 1 T T F <---
12862024-02-15T02:57:45.949Z 21 420-439 9 9 9 15 15 15 F F F
12872024-02-15T02:57:45.949Z 22 440-459 9 9 9 15 15 15 F F F
12882024-02-15T02:57:45.949Z 23 460-479 9 9 9 15 15 15 F F F
12892024-02-15T02:57:45.950Z 24 480-499 9 9 9 15 15 15 T T F <---
12902024-02-15T02:57:45.950Z 25 500-519 9 9 9 15 15 15 F F F
12912024-02-15T02:57:45.950Z 26 520-539 4 4 4 7 7 7 F F F
12922024-02-15T02:57:45.950Z 27 540-559 9 9 9 15 15 15 T T F <---
12932024-02-15T02:57:45.950Z 28 560-579 9 9 9 15 15 15 T T F <---
12942024-02-15T02:57:45.950Z 29 580-599 4 4 4 6 6 6 F F F
12952024-02-15T02:57:45.950ZMax gen: 10, Max flush: 18
12962024-02-15T02:57:45.951ZA Difference in extent metadata is expected here
12972024-02-15T02:57:45.951ZOn loop 5
12982024-02-15T02:57:45.951Z
12992024-02-15T02:57:45.951ZVerifying data now
13002024-02-15T02:57:46.837ZNo change after verify
13012024-02-15T02:57:46.971ZLoop: 5 Downstairs dump after verify (and repair):
13022024-02-15T02:57:46.971ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
13032024-02-15T02:57:46.972Z 0 000-019 8 8 8 13 13 13 F F F
13042024-02-15T02:57:46.972Z 1 020-039 11 11 11 19 19 19 F F F
13052024-02-15T02:57:46.972Z 2 040-059 8 8 8 14 14 14 F F F
13062024-02-15T02:57:46.972Z 3 060-079 11 11 11 19 19 19 F F F
13072024-02-15T02:57:46.972Z 4 080-099 4 4 4 5 5 5 F F F
13082024-02-15T02:57:46.972Z 5 100-119 7 7 7 12 12 12 F F F
13092024-02-15T02:57:46.972Z 6 120-139 3 3 3 3 3 3 F F F
13102024-02-15T02:57:46.973Z 7 140-159 6 6 6 11 11 11 F F F
13112024-02-15T02:57:46.973Z 8 160-179 4 4 4 7 7 7 F F F
13122024-02-15T02:57:46.973Z 9 180-199 11 11 11 19 19 19 F F F
13132024-02-15T02:57:46.973Z 10 200-219 11 11 11 19 19 19 F F F
13142024-02-15T02:57:46.973Z 11 220-239 11 11 11 19 19 19 F F F
13152024-02-15T02:57:46.973Z 12 240-259 10 10 10 16 16 16 F F F
13162024-02-15T02:57:46.973Z 13 260-279 9 9 9 15 15 15 F F F
13172024-02-15T02:57:46.973Z 14 280-299 1 1 1 1 1 1 F F F
13182024-02-15T02:57:46.974Z 15 300-319 10 10 10 18 18 18 F F F
13192024-02-15T02:57:46.974Z 16 320-339 9 9 9 15 15 15 F F F
13202024-02-15T02:57:46.974Z 17 340-359 9 9 9 15 15 15 F F F
13212024-02-15T02:57:46.974Z 18 360-379 5 5 5 8 8 8 F F F
13222024-02-15T02:57:46.974Z 19 380-399 9 9 9 15 15 15 F F F
13232024-02-15T02:57:46.974Z 20 400-419 11 11 11 19 19 19 F F F
13242024-02-15T02:57:46.974Z 21 420-439 9 9 9 15 15 15 F F F
13252024-02-15T02:57:46.974Z 22 440-459 9 9 9 15 15 15 F F F
13262024-02-15T02:57:46.974Z 23 460-479 9 9 9 15 15 15 F F F
13272024-02-15T02:57:46.974Z 24 480-499 11 11 11 19 19 19 F F F
13282024-02-15T02:57:46.974Z 25 500-519 9 9 9 15 15 15 F F F
13292024-02-15T02:57:46.974Z 26 520-539 4 4 4 7 7 7 F F F
13302024-02-15T02:57:46.974Z 27 540-559 11 11 11 19 19 19 F F F
13312024-02-15T02:57:46.974Z 28 560-579 11 11 11 19 19 19 F F F
13322024-02-15T02:57:46.974Z 29 580-599 4 4 4 6 6 6 F F F
13332024-02-15T02:57:46.974ZMax gen: 11, Max flush: 19
13342024-02-15T02:57:46.974Z
13352024-02-15T02:57:46.974ZBegin loop 6
13362024-02-15T02:57:46.974ZDownstairs to restart: 1
13372024-02-15T02:57:46.974Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.763593512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183}
13382024-02-15T02:57:46.974Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
13392024-02-15T02:57:46.974Z,"time":"2024-02-15T02:57:36.763981614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183}
13402024-02-15T02:57:46.974Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764029757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183}
13412024-02-15T02:57:46.974Z{"msg":"Crucible 2f62aa42-51b8-4cb7-91dd-fa3292fb849c has session id: be1948f9-95f0-4f08-89ba-bad37b7b97e2","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764084921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13422024-02-15T02:57:46.975Z{"msg":"Upstairs opts: Upstairs UUID: 2f62aa42-51b8-4cb7-91dd-fa3292fb849c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764144303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13432024-02-15T02:57:46.975Z{"msg":"Crucible stats registered with UUID: 2f62aa42-51b8-4cb7-91dd-fa3292fb849c","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764184298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13442024-02-15T02:57:46.975Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764222673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183}
13452024-02-15T02:57:46.975Z{"msg":"2f62aa42-51b8-4cb7-91dd-fa3292fb849c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764257989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13462024-02-15T02:57:46.975Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764281376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13472024-02-15T02:57:46.975Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764319131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13482024-02-15T02:57:46.975Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764362486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13492024-02-15T02:57:46.975Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.76440651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13502024-02-15T02:57:46.975Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764495009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13512024-02-15T02:57:46.975Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764525415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13522024-02-15T02:57:46.975Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764545343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13532024-02-15T02:57:46.975Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764577209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13542024-02-15T02:57:46.975Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764612335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13552024-02-15T02:57:46.975Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764641391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13562024-02-15T02:57:46.975Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764714582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13572024-02-15T02:57:46.975Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:36.76488822Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13582024-02-15T02:57:46.975Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:36.764933424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13592024-02-15T02:57:46.975Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:36.764952462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13602024-02-15T02:57:46.975Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.76496602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13612024-02-15T02:57:46.975Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.764980618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13622024-02-15T02:57:46.975Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.765081276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13632024-02-15T02:57:46.975Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.765423213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13642024-02-15T02:57:46.975Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.765461118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13652024-02-15T02:57:46.975Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.765564335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13662024-02-15T02:57:46.976Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.76568296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13672024-02-15T02:57:46.976Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.765963735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13682024-02-15T02:57:46.976Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.76616143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13692024-02-15T02:57:46.976Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.766596876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13702024-02-15T02:57:46.976Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.7666462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13712024-02-15T02:57:46.976Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.766670357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13722024-02-15T02:57:46.976Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.766705982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13732024-02-15T02:57:46.976Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.76672825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13742024-02-15T02:57:46.976Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:36.766743048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13752024-02-15T02:57:46.976Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.766577821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13762024-02-15T02:57:46.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.766663591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13772024-02-15T02:57:46.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.767504545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13782024-02-15T02:57:46.976Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.767535301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13792024-02-15T02:57:46.976Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768100201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13802024-02-15T02:57:46.976Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768642343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13812024-02-15T02:57:46.976Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768671039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13822024-02-15T02:57:46.976Z{"msg":"[0]R flush_numbers[0..12]: [13, 19, 14, 19, 5, 12, 3, 11, 7, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768688737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13832024-02-15T02:57:46.976Z{"msg":"[0]R generation[0..12]: [8, 11, 8, 11, 4, 7, 3, 6, 4, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768704005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13842024-02-15T02:57:46.976Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768732962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13852024-02-15T02:57:46.976Z{"msg":"[1]R flush_numbers[0..12]: [13, 19, 14, 19, 5, 12, 3, 11, 7, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768755819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13862024-02-15T02:57:46.976Z{"msg":"[1]R generation[0..12]: [8, 11, 8, 11, 4, 7, 3, 6, 4, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768771766Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13872024-02-15T02:57:46.976Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768793684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13882024-02-15T02:57:46.976Z{"msg":"[2]R flush_numbers[0..12]: [13, 19, 14, 19, 5, 12, 3, 11, 7, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768809072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13892024-02-15T02:57:46.976Z{"msg":"[2]R generation[0..12]: [8, 11, 8, 11, 4, 7, 3, 6, 4, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768830059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13902024-02-15T02:57:46.976Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768845217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13912024-02-15T02:57:46.977ZWait for a query_work_queue command to finish before sending IO
13922024-02-15T02:57:46.977Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768866325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13932024-02-15T02:57:46.977Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768888622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13942024-02-15T02:57:46.977Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.76890307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13952024-02-15T02:57:46.977Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768924058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13962024-02-15T02:57:46.977Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768938986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13972024-02-15T02:57:46.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768959643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13982024-02-15T02:57:46.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768974121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
13992024-02-15T02:57:46.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.768994839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
14002024-02-15T02:57:46.977Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.769026515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
14012024-02-15T02:57:47.017ZError: Difference in extent metadata found!
14022024-02-15T02:57:47.017Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.769047472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
14032024-02-15T02:57:47.017Z{"msg":"2f62aa42-51b8-4cb7-91dd-fa3292fb849c is now active with session: be1948f9-95f0-4f08-89ba-bad37b7b97e2","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.769067869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
14042024-02-15T02:57:47.017Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.769086677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183,"session_id":"be1948f9-95f0-4f08-89ba-bad37b7b97e2"}
14052024-02-15T02:57:47.017Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:37.769100795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183}
14062024-02-15T02:57:47.018ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
14072024-02-15T02:57:47.018ZLoading write count information from file "/tmp/test_repair_verify.data"
14082024-02-15T02:57:47.018ZRun Repair workload
14092024-02-15T02:57:47.018Z01/30 Write block 310 len 8192 data: 6 5
14102024-02-15T02:57:47.018Z02/30 Read block 240 len 16384
14112024-02-15T02:57:47.018Z03/30 Write block 586 len 40960 data: 2 2 2 3 2 2 2 2 3 2
14122024-02-15T02:57:47.018Z04/30 Read block 539 len 8192
14132024-02-15T02:57:47.018Z05/30 Write block 313 len 24576 data: 4 3 2 2 2 3
14142024-02-15T02:57:47.019Z06/30 Flush
14152024-02-15T02:57:47.019Z07/30 Write block 360 len 32768 data: 2 2 2 2 2 3 2 2
14162024-02-15T02:57:47.019Z08/30 Write block 240 len 24576 data: 2 3 4 4 4 4
14172024-02-15T02:57:47.019Z09/30 Read block 317 len 4096
14182024-02-15T02:57:47.019Z10/30 Flush
14192024-02-15T02:57:47.019Z11/30 Write block 573 len 12288 data: 3 2 2
14202024-02-15T02:57:47.019Z12/30 Write block 412 len 20480 data: 3 3 3 2 2
14212024-02-15T02:57:47.019Z13/30 Write block 87 len 4096 data: 2
14222024-02-15T02:57:47.019Z14/30 Write block 445 len 16384 data: 4 4 4 3
14232024-02-15T02:57:47.019Z15/30 Read block 515 len 28672
14242024-02-15T02:57:47.019Z16/30 Read block 2 len 28672
14252024-02-15T02:57:47.019Z17/30 Read block 185 len 32768
14262024-02-15T02:57:47.019Z18/30 Flush
14272024-02-15T02:57:47.019Z19/30 Write block 409 len 32768 data: 3 3 3 4 4 4 3 3
14282024-02-15T02:57:47.019Z20/30 Read block 586 len 40960
14292024-02-15T02:57:47.019Z21/30 Read block 278 len 36864
14302024-02-15T02:57:47.019Z22/30 Flush
14312024-02-15T02:57:47.019Z23/30 Flush
14322024-02-15T02:57:47.019Z24/30 Write block 254 len 12288 data: 2 2 2
14332024-02-15T02:57:47.020Z25/30 Write block 589 len 24576 data: 4 3 3 3 3 4
14342024-02-15T02:57:47.020Z26/30 Read block 491 len 28672
14352024-02-15T02:57:47.020Z27/30 Write block 133 len 20480 data: 3 3 2 2 2
14362024-02-15T02:57:47.020Z28/30 Read block 238 len 36864
14372024-02-15T02:57:47.020Z29/30 Read block 456 len 8192
14382024-02-15T02:57:47.020Z30/30 Read block 505 len 40960
14392024-02-15T02:57:47.020Z----------------------------------------------------------------
14402024-02-15T02:57:47.020Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14412024-02-15T02:57:47.020ZStates: Active Active Active
14422024-02-15T02:57:47.020ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14432024-02-15T02:57:47.020Z 1 Acked 1000 Write 2 Done Done Done false
14442024-02-15T02:57:47.020Z 2 Acked 1001 Read 4 Done Done Done false
14452024-02-15T02:57:47.020Z 3 Acked 1002 Write 10 Done Done Done false
14462024-02-15T02:57:47.020Z 4 Acked 1003 Read 2 Done Done Done false
14472024-02-15T02:57:47.020Z 5 Acked 1004 Write 6 Done Sent Done false
14482024-02-15T02:57:47.020Z 6 Acked 1005 Flush 0 Done Sent Done false
14492024-02-15T02:57:47.020Z 7 Acked 1006 Write 8 Done Sent Done false
14502024-02-15T02:57:47.020Z 8 Acked 1007 Write 6 Done Sent Done false
14512024-02-15T02:57:47.020Z 9 Acked 1008 Read 1 Done Sent Done false
14522024-02-15T02:57:47.020Z 10 Acked 1009 Flush 0 Done Sent Done false
14532024-02-15T02:57:47.020Z 11 Acked 1010 Write 3 Done Sent Done false
14542024-02-15T02:57:47.020Z 12 Acked 1011 Write 5 Done Sent Done false
14552024-02-15T02:57:47.021Z 13 Acked 1012 Write 1 Done Sent Done false
14562024-02-15T02:57:47.021Z 14 Acked 1013 Write 4 Done Sent Done false
14572024-02-15T02:57:47.021Z 15 Acked 1014 Read 7 Done Sent Done false
14582024-02-15T02:57:47.021Z 16 Acked 1015 Read 7 Done Sent Done false
14592024-02-15T02:57:47.021Z 17 Acked 1016 Read 8 Done Sent Done false
14602024-02-15T02:57:47.021Z 18 Acked 1017 Flush 0 Done Sent Done false
14612024-02-15T02:57:47.021Z 19 Acked 1018 Write 8 Done Sent Done false
14622024-02-15T02:57:47.021Z 20 Acked 1019 Read 10 Done Sent Done false
14632024-02-15T02:57:47.021Z 21 Acked 1020 Read 9 Done Sent Done false
14642024-02-15T02:57:47.021Z 22 Acked 1021 Flush 0 Done Sent Done false
14652024-02-15T02:57:47.021Z 23 Acked 1022 Flush 0 Done Sent Done false
14662024-02-15T02:57:47.021Z 24 Acked 1023 Write 3 Done Sent Done false
14672024-02-15T02:57:47.021Z 25 Acked 1024 Write 6 Done Sent Done false
14682024-02-15T02:57:47.021Z 26 Acked 1025 Read 7 Done Sent Done false
14692024-02-15T02:57:47.021Z 27 Acked 1026 Write 5 Done Sent Done false
14702024-02-15T02:57:47.021Z 28 Acked 1027 Read 9 Done Sent Done false
14712024-02-15T02:57:47.021Z 29 Acked 1028 Read 2 Done Sent Done false
14722024-02-15T02:57:47.021Z 30 Acked 1029 Read 10 Done Sent Sent false
14732024-02-15T02:57:47.021Z STATES DS:0 DS:1 DS:2 TOTAL
14742024-02-15T02:57:47.021Z New 0 0 0 0
14752024-02-15T02:57:47.021Z Sent 0 26 1 27
14762024-02-15T02:57:47.021Z Done 30 4 29 63
14772024-02-15T02:57:47.021Z Skipped 0 0 0 0
14782024-02-15T02:57:47.021Z Error 0 0 0 0
14792024-02-15T02:57:47.022ZLast Flush: 1022 0 1022
14802024-02-15T02:57:47.022ZDownstairs last five completed:
14812024-02-15T02:57:47.022ZUpstairs last five completed: 30 29 28 27 26
14822024-02-15T02:57:47.022Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:37.800241563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1183}
14832024-02-15T02:57:47.022ZWrote out file "/tmp/test_repair_verify.data"
14842024-02-15T02:57:47.022Z
14852024-02-15T02:57:47.022ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14862024-02-15T02:57:47.022ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14872024-02-15T02:57:47.022Z 0 000-019 8 8 8 13 13 13 F F F
14882024-02-15T02:57:47.022Z 1 020-039 11 11 11 19 19 19 F F F
14892024-02-15T02:57:47.022Z 2 040-059 8 8 8 14 14 14 F F F
14902024-02-15T02:57:47.022Z 3 060-079 11 11 11 19 19 19 F F F
14912024-02-15T02:57:47.022Z 4 080-099 12 4 12 22 5 22 F F F <---
14922024-02-15T02:57:47.022Z 5 100-119 7 7 7 12 12 12 F F F
14932024-02-15T02:57:47.022Z 6 120-139 3 3 3 3 3 3 T F T <---
14942024-02-15T02:57:47.022Z 7 140-159 6 6 6 11 11 11 F F F
14952024-02-15T02:57:47.022Z 8 160-179 4 4 4 7 7 7 F F F
14962024-02-15T02:57:47.022Z 9 180-199 11 11 11 19 19 19 F F F
14972024-02-15T02:57:47.022Z 10 200-219 11 11 11 19 19 19 F F F
14982024-02-15T02:57:47.022Z 11 220-239 11 11 11 19 19 19 F F F
14992024-02-15T02:57:47.022Z 12 240-259 12 10 12 21 16 21 T F T <---
15002024-02-15T02:57:47.022Z 13 260-279 9 9 9 15 15 15 F F F
15012024-02-15T02:57:47.051Z 14 280-299 1 1 1 1 1 1 F F F
15022024-02-15T02:57:47.051Z 15 300-319 12 10 12 20 18 20 F T F <---
15032024-02-15T02:57:47.051Z 16 320-339 9 9 9 15 15 15 F F F
15042024-02-15T02:57:47.051Z 17 340-359 9 9 9 15 15 15 F F F
15052024-02-15T02:57:47.051Z 18 360-379 12 5 12 21 8 21 F F F <---
15062024-02-15T02:57:47.051Z 19 380-399 9 9 9 15 15 15 F F F
15072024-02-15T02:57:47.051Z 20 400-419 12 11 12 23 19 23 F F F <---
15082024-02-15T02:57:47.051Z 21 420-439 9 9 9 15 15 15 F F F
15092024-02-15T02:57:47.051Z 22 440-459 12 9 12 22 15 22 F F F <---
15102024-02-15T02:57:47.051Z 23 460-479 9 9 9 15 15 15 F F F
15112024-02-15T02:57:47.052Z 24 480-499 11 11 11 19 19 19 F F F
15122024-02-15T02:57:47.052Z 25 500-519 9 9 9 15 15 15 F F F
15132024-02-15T02:57:47.052Z 26 520-539 4 4 4 7 7 7 F F F
15142024-02-15T02:57:47.052Z 27 540-559 11 11 11 19 19 19 F F F
15152024-02-15T02:57:47.052Z 28 560-579 12 11 12 22 19 22 F F F <---
15162024-02-15T02:57:47.052Z 29 580-599 12 4 12 20 6 20 T T T <---
15172024-02-15T02:57:47.052ZMax gen: 12, Max flush: 23
15182024-02-15T02:57:47.052ZA Difference in extent metadata is expected here
15192024-02-15T02:57:47.052ZOn loop 6
15202024-02-15T02:57:47.052Z
15212024-02-15T02:57:47.052ZVerifying data now
15222024-02-15T02:57:47.052ZNo change after verify
15232024-02-15T02:57:47.052ZLoop: 6 Downstairs dump after verify (and repair):
15242024-02-15T02:57:47.052ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
15252024-02-15T02:57:47.052Z 0 000-019 8 8 8 13 13 13 F F F
15262024-02-15T02:57:47.053Z 1 020-039 11 11 11 19 19 19 F F F
15272024-02-15T02:57:47.053Z 2 040-059 8 8 8 14 14 14 F F F
15282024-02-15T02:57:47.053Z 3 060-079 11 11 11 19 19 19 F F F
15292024-02-15T02:57:47.053Z 4 080-099 12 12 12 22 22 22 F F F
15302024-02-15T02:57:47.053Z 5 100-119 7 7 7 12 12 12 F F F
15312024-02-15T02:57:47.053Z 6 120-139 13 13 13 24 24 24 F F F
15322024-02-15T02:57:47.053Z 7 140-159 6 6 6 11 11 11 F F F
15332024-02-15T02:57:47.053Z 8 160-179 4 4 4 7 7 7 F F F
15342024-02-15T02:57:47.053Z 9 180-199 11 11 11 19 19 19 F F F
15352024-02-15T02:57:47.053Z 10 200-219 11 11 11 19 19 19 F F F
15362024-02-15T02:57:47.053Z 11 220-239 11 11 11 19 19 19 F F F
15372024-02-15T02:57:47.053Z 12 240-259 13 13 13 24 24 24 F F F
15382024-02-15T02:57:47.053Z 13 260-279 9 9 9 15 15 15 F F F
15392024-02-15T02:57:47.053Z 14 280-299 1 1 1 1 1 1 F F F
15402024-02-15T02:57:47.053Z 15 300-319 12 12 12 20 20 20 F F F
15412024-02-15T02:57:47.053Z 16 320-339 9 9 9 15 15 15 F F F
15422024-02-15T02:57:47.053Z 17 340-359 9 9 9 15 15 15 F F F
15432024-02-15T02:57:47.053Z 18 360-379 12 12 12 21 21 21 F F F
15442024-02-15T02:57:47.053Z 19 380-399 9 9 9 15 15 15 F F F
15452024-02-15T02:57:47.053Z 20 400-419 12 12 12 23 23 23 F F F
15462024-02-15T02:57:47.053Z 21 420-439 9 9 9 15 15 15 F F F
15472024-02-15T02:57:47.053Z 22 440-459 12 12 12 22 22 22 F F F
15482024-02-15T02:57:47.053Z 23 460-479 9 9 9 15 15 15 F F F
15492024-02-15T02:57:47.054Z 24 480-499 11 11 11 19 19 19 F F F
15502024-02-15T02:57:47.054Z 25 500-519 9 9 9 15 15 15 F F F
15512024-02-15T02:57:47.054Z 26 520-539 4 4 4 7 7 7 F F F
15522024-02-15T02:57:47.054Z 27 540-559 11 11 11 19 19 19 F F F
15532024-02-15T02:57:47.054Z 28 560-579 12 12 12 22 22 22 F F F
15542024-02-15T02:57:47.054Z 29 580-599 13 13 13 24 24 24 F F F
15552024-02-15T02:57:47.054ZMax gen: 13, Max flush: 24
15562024-02-15T02:57:47.054Z
15572024-02-15T02:57:47.054ZBegin loop 7
15582024-02-15T02:57:47.054ZDownstairs to restart: 0
15592024-02-15T02:57:47.054Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.341561183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191}
15602024-02-15T02:57:47.054Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.34182178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":Crucible runtime is spawned
15612024-02-15T02:57:47.054Z1191}
15622024-02-15T02:57:47.054Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.341856676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191}
15632024-02-15T02:57:47.054Z{"msg":"Crucible 7bee40e9-bcc3-4e4e-8247-21a83d16625f has session id: a34aa65a-f06a-4fa5-8842-204546a83436","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.341878623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15642024-02-15T02:57:47.054Z{"msg":"Upstairs opts: Upstairs UUID: 7bee40e9-bcc3-4e4e-8247-21a83d16625f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.341913689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15652024-02-15T02:57:47.054Z{"msg":"Crucible stats registered with UUID: 7bee40e9-bcc3-4e4e-8247-21a83d16625f","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.341932587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15662024-02-15T02:57:47.054Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.341945525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191}
15672024-02-15T02:57:47.054Z{"msg":"7bee40e9-bcc3-4e4e-8247-21a83d16625f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342139811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15682024-02-15T02:57:47.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342156829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15692024-02-15T02:57:47.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342171517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15702024-02-15T02:57:47.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342185795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15712024-02-15T02:57:47.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342203393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15722024-02-15T02:57:47.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.34222706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15732024-02-15T02:57:47.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342253166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15742024-02-15T02:57:47.055Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342279523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15752024-02-15T02:57:47.055Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342311389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15762024-02-15T02:57:47.055Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342338916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15772024-02-15T02:57:47.055Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342456691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15782024-02-15T02:57:47.055Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342497256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15792024-02-15T02:57:47.055Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:43.342526362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15802024-02-15T02:57:47.055Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:43.34254383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15812024-02-15T02:57:47.055Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:43.342613711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15822024-02-15T02:57:47.055Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342632219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15832024-02-15T02:57:47.055Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.342646107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15842024-02-15T02:57:47.055Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.34277958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15852024-02-15T02:57:47.055Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.343195099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15862024-02-15T02:57:47.055Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.343223045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15872024-02-15T02:57:47.055Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:43.343239323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15882024-02-15T02:57:47.056ZWait for a query_work_queue command to finish before sending IO
15892024-02-15T02:57:47.056Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.574884214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15902024-02-15T02:57:47.056Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.574941887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15912024-02-15T02:57:47.056Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.574963754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15922024-02-15T02:57:47.056Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.574982162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15932024-02-15T02:57:47.056Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575004709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15942024-02-15T02:57:47.056Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575033955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15952024-02-15T02:57:47.056Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575051313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15962024-02-15T02:57:47.056Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575068591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15972024-02-15T02:57:47.056Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575085869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15982024-02-15T02:57:47.056Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575101317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
15992024-02-15T02:57:47.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575119654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16002024-02-15T02:57:47.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575138072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16012024-02-15T02:57:47.099Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.57515524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16022024-02-15T02:57:47.099Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575173327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16032024-02-15T02:57:47.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575190305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16042024-02-15T02:57:47.099Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575206993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16052024-02-15T02:57:47.099Z{"msg":"[0]R flush_numbers[0..12]: [13, 19, 14, 19, 22, 12, 24, 11, 7, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575224471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16062024-02-15T02:57:47.099Z{"msg":"[0]R generation[0..12]: [8, 11, 8, 11, 12, 7, 13, 6, 4, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575242389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16072024-02-15T02:57:47.100Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575260347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16082024-02-15T02:57:47.100Z{"msg":"[1]R flush_numbers[0..12]: [13, 19, 14, 19, 22, 12, 24, 11, 7, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575277894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16092024-02-15T02:57:47.100Z{"msg":"[1]R generation[0..12]: [8, 11, 8, 11, 12, 7, 13, 6, 4, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575294892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16102024-02-15T02:57:47.100Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.57531302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16112024-02-15T02:57:47.100Z{"msg":"[2]R flush_numbers[0..12]: [13, 19, 14, 19, 22, 12, 24, 11, 7, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575330588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16122024-02-15T02:57:47.100Z{"msg":"[2]R generation[0..12]: [8, 11, 8, 11, 12, 7, 13, 6, 4, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575348046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16132024-02-15T02:57:47.100Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575365834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16142024-02-15T02:57:47.100Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575382621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16152024-02-15T02:57:47.100Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575400029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16162024-02-15T02:57:47.100Z{"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.575415957Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
16172024-02-15T02:57:47.100Zip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16182024-02-15T02:57:47.100Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.793775357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16192024-02-15T02:57:47.100Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.793831881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16202024-02-15T02:57:47.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.793860077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16212024-02-15T02:57:47.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.793891123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16222024-02-15T02:57:47.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.79391724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16232024-02-15T02:57:47.101Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.793944956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16242024-02-15T02:57:47.101Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.793970663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16252024-02-15T02:57:47.101Z{"msg":"7bee40e9-bcc3-4e4e-8247-21a83d16625f is now active with session: a34aa65a-f06a-4fa5-8842-204546a83436","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.794011598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16262024-02-15T02:57:47.101Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.794038764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191,"session_id":"a34aa65a-f06a-4fa5-8842-204546a83436"}
16272024-02-15T02:57:47.101Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:45.794063241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191}
16282024-02-15T02:57:47.101ZLoading write count information from file "/tmp/test_repair_verify.data"
16292024-02-15T02:57:47.101ZRun Repair workload
16302024-02-15T02:57:47.101Z01/30 Write block 126 len 8192 data: 3 3
16312024-02-15T02:57:47.101Z02/30 Read block 528 len 36864
16322024-02-15T02:57:47.101Z03/30 Write block 156 len 28672 data: 2 2 2 2 2 2 2
16332024-02-15T02:57:47.102Z04/30 Read block 365 len 40960
16342024-02-15T02:57:47.102Z05/30 Read block 182 len 20480
16352024-02-15T02:57:47.102Z06/30 Write block 511 len 24576 data: 3 4 4 4 4 4
16362024-02-15T02:57:47.102Z07/30 Read block 130 len 20480
16372024-02-15T02:57:47.102Z08/30 Read block 230 len 12288
16382024-02-15T02:57:47.102Z09/30 Read block 256 len 20480
16392024-02-15T02:57:47.102Z10/30 Write block 97 len 36864 data: 3 3 2 2 2 2 2 3 3
16402024-02-15T02:57:47.102Z11/30 Write block 8 len 28672 data: 4 4 5 3 3 2 2
16412024-02-15T02:57:47.102Z12/30 Flush
16422024-02-15T02:57:47.102Z13/30 Flush
16432024-02-15T02:57:47.102Z14/30 Write block 147 len 20480 data: 2 2 2 2 2
16442024-02-15T02:57:47.102Z15/30 Read block 48 len 4096
16452024-02-15T02:57:47.102Z16/30 Read block 580 len 36864
16462024-02-15T02:57:47.103Z17/30 Write block 94 len 24576 data: 4 3 3 4 4 3
16472024-02-15T02:57:47.103Z18/30 Read block 434 len 28672
16482024-02-15T02:57:47.103Z19/30 Read block 304 len 28672
16492024-02-15T02:57:47.103Z20/30 Read block 265 len 32768
16502024-02-15T02:57:47.103Z21/30 Flush
16512024-02-15T02:57:47.103Z22/30 Write block 478 len 8192 data: 2 2
16522024-02-15T02:57:47.103Z23/30 Write block 206 len 28672 data: 2 2 2 2 2 3 3
16532024-02-15T02:57:47.103Z24/30 Read block 495 len 12288
16542024-02-15T02:57:47.103Z25/30 Read block 418 len 40960
16552024-02-15T02:57:47.103Z26/30 Write block 252 len 40960 data: 2 2 3 3 3 2 2 3 3 3
16562024-02-15T02:57:47.103Z27/30 Read block 541 len 36864
16572024-02-15T02:57:47.103Z28/30 Write block 376 len 28672 data: 2 2 3 3 3 3 3
16582024-02-15T02:57:47.103Z29/30 Read block 346 len 20480
16592024-02-15T02:57:47.104Z30/30 Read block 251 len 8192
16602024-02-15T02:57:47.104Z----------------------------------------------------------------
16612024-02-15T02:57:47.104Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16622024-02-15T02:57:47.104ZStates: Active Active Active
16632024-02-15T02:57:47.104ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16642024-02-15T02:57:47.104Z 1 Acked 1000 Write 2 Done Done Done false
16652024-02-15T02:57:47.104Z 2 Acked 1001 Read 9 Done Done Done false
16662024-02-15T02:57:47.104Z 3 Acked 1002 Write 7 Sent Done Done false
16672024-02-15T02:57:47.104Z 4 Acked 1003 Read 10 Sent Done Done false
16682024-02-15T02:57:47.104Z 5 Acked 1004 Read 5 Sent Done Done false
16692024-02-15T02:57:47.104Z 6 Acked 1005 Write 6 Sent Done Done false
16702024-02-15T02:57:47.105Z 7 Acked 1006 Read 5 Sent Done Done false
16712024-02-15T02:57:47.105Z 8 Acked 1007 Read 3 Sent Done Done false
16722024-02-15T02:57:47.105Z 9 Acked 1008 Read 5 Sent Done Done false
16732024-02-15T02:57:47.105Z 10 Acked 1009 Write 9 Sent Done Done false
16742024-02-15T02:57:47.105Z 11 Acked 1010 Write 7 Sent Done Done false
16752024-02-15T02:57:47.105Z 12 Acked 1011 Flush 0 Sent Done Done false
16762024-02-15T02:57:47.105Z 13 Acked 1012 Flush 0 Sent Done Done false
16772024-02-15T02:57:47.105Z 14 Acked 1013 Write 5 Sent Done Done false
16782024-02-15T02:57:47.105Z 15 Acked 1014 Read 1 Sent Done Done false
16792024-02-15T02:57:47.105Z 16 Acked 1015 Read 9 Sent Done Done false
16802024-02-15T02:57:47.105Z 17 Acked 1016 Write 6 Sent Done Done false
16812024-02-15T02:57:47.105Z 18 Acked 1017 Read 7 Sent Done Done false
16822024-02-15T02:57:47.106Z 19 Acked 1018 Read 7 Sent Done Done false
16832024-02-15T02:57:47.106Z 20 Acked 1019 Read 8 Sent Done Done false
16842024-02-15T02:57:47.106Z 21 Acked 1020 Flush 0 Sent Done Done false
16852024-02-15T02:57:47.106Z 22 Acked 1021 Write 2 Sent Done Done false
16862024-02-15T02:57:47.106Z 23 Acked 1022 Write 7 Sent Done Done false
16872024-02-15T02:57:47.106Z 24 Acked 1023 Read 3 Sent Done Done false
16882024-02-15T02:57:47.106Z 25 Acked 1024 Read 10 Sent Done Done false
16892024-02-15T02:57:47.107Z 26 Acked 1025 Write 10 Sent Done Done false
16902024-02-15T02:57:47.107Z 27 Acked 1026 Read 9 Sent Done Done false
16912024-02-15T02:57:47.107Z 28 Acked 1027 Write 7 Sent Done Done false
16922024-02-15T02:57:47.107Z 29 Acked 1028 Read 5 Sent Done Done false
16932024-02-15T02:57:47.107Z 30 Acked 1029 Read 2 Sent Done Sent false
16942024-02-15T02:57:47.107Z STATES DS:0 DS:1 DS:2 TOTAL
16952024-02-15T02:57:47.108Z New 0 0 0 0
16962024-02-15T02:57:47.108Z Sent 28 0 1 29
16972024-02-15T02:57:47.108Z Done 2 30 29 61
16982024-02-15T02:57:47.108Z Skipped 0 0 0 0
16992024-02-15T02:57:47.108Z Error 0 0 0 0
17002024-02-15T02:57:47.108ZLast Flush: 0 1020 1020
17012024-02-15T02:57:47.129ZDownstairs last five completed:
17022024-02-15T02:57:47.130ZUpstairs last five completed: 30 29 28 27 26
17032024-02-15T02:57:47.130Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:45.827022342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1191}
17042024-02-15T02:57:47.130ZWrote out file "/tmp/test_repair_verify.data"
17052024-02-15T02:57:47.130Z
17062024-02-15T02:57:47.130ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
17072024-02-15T02:57:47.130ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17082024-02-15T02:57:47.130Z 0 000-019 8 14 14 13 25 25 F F F <---
17092024-02-15T02:57:47.130Z 1 020-039 11 11 11 19 19 19 F F F
17102024-02-15T02:57:47.130Z 2 040-059 8 8 8 14 14 14 F F F
17112024-02-15T02:57:47.130Z 3 060-079 11 11 11 19 19 19 F F F
17122024-02-15T02:57:47.130Z 4 080-099 12 14 14 22 27 27 F F F <---
17132024-02-15T02:57:47.131Z 5 100-119 7 14 14 12 25 25 F F F <---
17142024-02-15T02:57:47.131Z 6 120-139 13 14 14 24 25 25 T F F <---
17152024-02-15T02:57:47.131Z 7 140-159 6 14 14 11 27 27 F F F <---
17162024-02-15T02:57:47.131Z 8 160-179 4 14 14 7 25 25 F F F <---
17172024-02-15T02:57:47.131Z 9 180-199 11 11 11 19 19 19 F F F
17182024-02-15T02:57:47.131Z 10 200-219 11 11 11 19 19 19 F T T <---
17192024-02-15T02:57:47.131Z 11 220-239 11 11 11 19 19 19 F F F
17202024-02-15T02:57:47.131Z 12 240-259 13 13 13 24 24 24 F T T <---
17212024-02-15T02:57:47.131Z 13 260-279 9 9 9 15 15 15 F T T <---
17222024-02-15T02:57:47.131Z 14 280-299 1 1 1 1 1 1 F F F
17232024-02-15T02:57:47.131Z 15 300-319 12 12 12 20 20 20 F F F
17242024-02-15T02:57:47.132Z 16 320-339 9 9 9 15 15 15 F F F
17252024-02-15T02:57:47.132Z 17 340-359 9 9 9 15 15 15 F F F
17262024-02-15T02:57:47.132Z 18 360-379 12 12 12 21 21 21 F T T <---
17272024-02-15T02:57:47.132Z 19 380-399 9 9 9 15 15 15 F T T <---
17282024-02-15T02:57:47.132Z 20 400-419 12 12 12 23 23 23 F F F
17292024-02-15T02:57:47.132Z 21 420-439 9 9 9 15 15 15 F F F
17302024-02-15T02:57:47.132Z 22 440-459 12 12 12 22 22 22 F F F
17312024-02-15T02:57:47.132Z 23 460-479 9 9 9 15 15 15 F T T <---
17322024-02-15T02:57:47.132Z 24 480-499 11 11 11 19 19 19 F F F
17332024-02-15T02:57:47.132Z 25 500-519 9 14 14 15 25 25 F F F <---
17342024-02-15T02:57:47.133Z 26 520-539 4 4 4 7 7 7 F F F
17352024-02-15T02:57:47.133Z 27 540-559 11 11 11 19 19 19 F F F
17362024-02-15T02:57:47.133Z 28 560-579 12 12 12 22 22 22 F F F
17372024-02-15T02:57:47.133Z 29 580-599 13 13 13 24 24 24 F F F
17382024-02-15T02:57:47.133ZMax gen: 14, Max flush: 27
17392024-02-15T02:57:47.133ZError: Difference in extent metadata found!
17402024-02-15T02:57:47.133ZA Difference in extent metadata is expected here
17412024-02-15T02:57:47.133ZOn loop 7
17422024-02-15T02:57:47.133Z
17432024-02-15T02:57:47.133ZVerifying data now
17442024-02-15T02:57:52.560ZNo change after verify
17452024-02-15T02:57:52.561ZLoop: 7 Downstairs dump after verify (and repair):
17462024-02-15T02:57:52.576ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17472024-02-15T02:57:52.576Z 0 000-019 14 14 14 25 25 25 F F F
17482024-02-15T02:57:52.576Z 1 020-039 11 11 11 19 19 19 F F F
17492024-02-15T02:57:52.576Z 2 040-059 8 8 8 14 14 14 F F F
17502024-02-15T02:57:52.576Z 3 060-079 11 11 11 19 19 19 F F F
17512024-02-15T02:57:52.577Z 4 080-099 14 14 14 27 27 27 F F F
17522024-02-15T02:57:52.577Z 5 100-119 14 14 14 25 25 25 F F F
17532024-02-15T02:57:52.577Z 6 120-139 14 14 14 25 25 25 F F F
17542024-02-15T02:57:52.577Z 7 140-159 14 14 14 27 27 27 F F F
17552024-02-15T02:57:52.577Z 8 160-179 14 14 14 25 25 25 F F F
17562024-02-15T02:57:52.577Z 9 180-199 11 11 11 19 19 19 F F F
17572024-02-15T02:57:52.577Z 10 200-219 15 15 15 28 28 28 F F F
17582024-02-15T02:57:52.577Z 11 220-239 11 11 11 19 19 19 F F F
17592024-02-15T02:57:52.578Z 12 240-259 15 15 15 28 28 28 F F F
17602024-02-15T02:57:52.578Z 13 260-279 15 15 15 28 28 28 F F F
17612024-02-15T02:57:52.578Z 14 280-299 1 1 1 1 1 1 F F F
17622024-02-15T02:57:52.593Z 15 300-319 12 12 12 20 20 20 F F F
17632024-02-15T02:57:52.594Z 16 320-339 9 9 9 15 15 15 F F F
17642024-02-15T02:57:52.594Z 17 340-359 9 9 9 15 15 15 F F F
17652024-02-15T02:57:52.594Z 18 360-379 15 15 15 28 28 28 F F F
17662024-02-15T02:57:52.594Z 19 380-399 15 15 15 28 28 28 F F F
17672024-02-15T02:57:52.594Z 20 400-419 12 12 12 23 23 23 F F F
17682024-02-15T02:57:52.595Z 21 420-439 9 9 9 15 15 15 F F F
17692024-02-15T02:57:52.595Z 22 440-459 12 12 12 22 22 22 F F F
17702024-02-15T02:57:52.595Z 23 460-479 15 15 15 28 28 28 F F F
17712024-02-15T02:57:52.595Z 24 480-499 11 11 11 19 19 19 F F F
17722024-02-15T02:57:52.595Z 25 500-519 14 14 14 25 25 25 F F F
17732024-02-15T02:57:52.596Z 26 520-539 4 4 4 7 7 7 F F F
17742024-02-15T02:57:52.596Z 27 540-559 11 11 11 19 19 19 F F F
17752024-02-15T02:57:52.596Z 28 560-579 12 12 12 22 22 22 F F F
17762024-02-15T02:57:52.597Z 29 580-599 13 13 13 24 24 24 F F F
17772024-02-15T02:57:52.597ZMax gen: 15, Max flush: 28
17782024-02-15T02:57:52.597Z
17792024-02-15T02:57:52.597ZBegin loop 8
17802024-02-15T02:57:52.598ZDownstairs to restart: 0
17812024-02-15T02:57:52.615Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549199851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199}
17822024-02-15T02:57:52.616Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549462707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199}
17832024-02-15T02:57:52.616Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549480835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199}
17842024-02-15T02:57:52.616Z{"msg":"Crucible 341e5fb8-5b9c-4646-8012-e4d77ff3cb41 has session id: 639cf80b-d75d-4e34-8e7c-c06dc66f88fa","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549494324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17852024-02-15T02:57:52.616Z{"msg":"Upstairs opts: Upstairs UUID: 341e5fb8-5b9c-4646-8012-e4d77ff3cb41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.5495247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17862024-02-15T02:57:52.616ZCrucible runtime is spawned
17872024-02-15T02:57:52.617Z{"msg":"Crucible stats registered with UUID: 341e5fb8-5b9c-4646-8012-e4d77ff3cb41","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549572894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17882024-02-15T02:57:52.617Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549656563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199}
17892024-02-15T02:57:52.617Z{"msg":"341e5fb8-5b9c-4646-8012-e4d77ff3cb41 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549798116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17902024-02-15T02:57:52.618Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549827562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17912024-02-15T02:57:52.618Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549854049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17922024-02-15T02:57:52.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549890064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17932024-02-15T02:57:52.619Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.54991745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17942024-02-15T02:57:52.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549942227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17952024-02-15T02:57:52.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549966175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17962024-02-15T02:57:52.620Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.549998291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17972024-02-15T02:57:52.620Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550046854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17982024-02-15T02:57:52.620Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.55008217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
17992024-02-15T02:57:52.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550107057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18002024-02-15T02:57:52.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550141293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18012024-02-15T02:57:52.621Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:51.550390131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18022024-02-15T02:57:52.622Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:51.550438875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18032024-02-15T02:57:52.622Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:51.55048047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18042024-02-15T02:57:52.622Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550508167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18052024-02-15T02:57:52.622Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550533184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18062024-02-15T02:57:52.623Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550648629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18072024-02-15T02:57:52.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550810119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18082024-02-15T02:57:52.623Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550854043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18092024-02-15T02:57:52.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.55087922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18102024-02-15T02:57:52.624Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.550928654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18112024-02-15T02:57:52.624Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.55143689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18122024-02-15T02:57:52.624Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.551576273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18132024-02-15T02:57:52.624Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.551927419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18142024-02-15T02:57:52.625Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.551964794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18152024-02-15T02:57:52.625Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.552021767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18162024-02-15T02:57:52.625Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.55215855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18172024-02-15T02:57:52.625Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.552195065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18182024-02-15T02:57:52.626Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:51.552220332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18192024-02-15T02:57:53.599Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.552282888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18202024-02-15T02:57:53.599Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.552471924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18212024-02-15T02:57:53.604Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.55361623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18222024-02-15T02:57:53.604Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.553679242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18232024-02-15T02:57:53.604Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554161312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18242024-02-15T02:57:53.604Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554780435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18252024-02-15T02:57:53.604Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554830848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18262024-02-15T02:57:53.604Z{"msg":"[0]R flush_numbers[0..12]: [25, 19, 14, 19, 27, 25, 25, 27, 25, 19, 28, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554880442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18272024-02-15T02:57:53.604Z{"msg":"[0]R generation[0..12]: [14, 11, 8, 11, 14, 14, 14, 14, 14, 11, 15, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554918917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18282024-02-15T02:57:53.605Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554938395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18292024-02-15T02:57:53.605Z{"msg":"[1]R flush_numbers[0..12]: [25, 19, 14, 19, 27, 25, 25, 27, 25, 19, 28, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554964232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18302024-02-15T02:57:53.605Z{"msg":"[1]R generation[0..12]: [14, 11, 8, 11, 14, 14, 14, 14, 14, 11, 15, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.554982729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18312024-02-15T02:57:53.605Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
18322024-02-15T02:57:53.605Z,"time":"2024-02-15T02:57:52.555024034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18332024-02-15T02:57:53.605Z{"msg":"[2]R flush_numbers[0..12]: [25, 19, 14, 19, 27, 25, 25, 27, 25, 19, 28, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555047811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18342024-02-15T02:57:53.605Z{"msg":"[2]R generation[0..12]: [14, 11, 8, 11, 14, 14, 14, 14, 14, 11, 15, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555086846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18352024-02-15T02:57:53.605Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555110813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18362024-02-15T02:57:53.605Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555147149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18372024-02-15T02:57:53.605Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555166206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18382024-02-15T02:57:53.605Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555204071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18392024-02-15T02:57:53.605Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555225319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18402024-02-15T02:57:53.605Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555261554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18412024-02-15T02:57:53.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555278812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18422024-02-15T02:57:53.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555302799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18432024-02-15T02:57:53.605Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555337585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18442024-02-15T02:57:53.605Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.55537479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18452024-02-15T02:57:53.605Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555421344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18462024-02-15T02:57:53.605Z{"msg":"341e5fb8-5b9c-4646-8012-e4d77ff3cb41 is now active with session: 639cf80b-d75d-4e34-8e7c-c06dc66f88fa","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555460719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18472024-02-15T02:57:53.605ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18482024-02-15T02:57:53.605Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.555500854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199,"session_id":"639cf80b-d75d-4e34-8e7c-c06dc66f88fa"}
18492024-02-15T02:57:53.605Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:52.55553886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199}
18502024-02-15T02:57:53.605ZLoading write count information from file "/tmp/test_repair_verify.data"
18512024-02-15T02:57:53.605ZRun Repair workload
18522024-02-15T02:57:53.610Z01/30 Write block 233 len 16384 data: 3 3 2 2
18532024-02-15T02:57:53.611Z02/30 Write block 557 len 12288 data: 4 4 5
18542024-02-15T02:57:53.611Z03/30 Read block 149 len 28672
18552024-02-15T02:57:53.611Z04/30 Flush
18562024-02-15T02:57:53.611Z05/30 Write block 501 len 24576 data: 2 2 2 2 2 2
18572024-02-15T02:57:53.611Z06/30 Write block 411 len 28672 data: 4 5 5 5 4 4 2
18582024-02-15T02:57:53.611Z07/30 Read block 159 len 20480
18592024-02-15T02:57:53.611Z08/30 Flush
18602024-02-15T02:57:53.618Z09/30 Write block 226 len 28672 data: 3 3 3 3 2 2 3
18612024-02-15T02:57:53.618Z10/30 Read block 146 len 40960
18622024-02-15T02:57:53.618Z11/30 Read block 208 len 40960
18632024-02-15T02:57:53.618Z12/30 Read block 284 len 20480
18642024-02-15T02:57:53.618Z13/30 Write block 299 len 28672 data: 2 3 2 2 3 5 6
18652024-02-15T02:57:53.618Z14/30 Write block 284 len 8192 data: 2 2
18662024-02-15T02:57:53.618Z15/30 Write block 81 len 8192 data: 2 2
18672024-02-15T02:57:53.623Z16/30 Write block 35 len 32768 data: 4 2 2 3 2 2 2 2
18682024-02-15T02:57:53.623Z17/30 Read block 251 len 24576
18692024-02-15T02:57:53.623Z18/30 Read block 279 len 28672
18702024-02-15T02:57:53.623Z19/30 Read block 535 len 32768
18712024-02-15T02:57:53.624Z20/30 Flush
18722024-02-15T02:57:53.631Z21/30 Flush
18732024-02-15T02:57:53.632Z22/30 Write block 123 len 28672 data: 2 2 3 4 4 3 3
18742024-02-15T02:57:53.632Z23/30 Read block 570 len 20480
18752024-02-15T02:57:53.632Z24/30 Write block 301 len 12288 data: 3 3 4
18762024-02-15T02:57:53.632Z25/30 Flush
18772024-02-15T02:57:53.632Z26/30 Flush
18782024-02-15T02:57:53.632Z27/30 Write block 542 len 16384 data: 4 3 2 2
18792024-02-15T02:57:53.638Z28/30 Write block 175 len 24576 data: 2 2 2 2 2 2
18802024-02-15T02:57:53.638Z29/30 Write block 189 len 32768 data: 3 2 2 3 4 4 4 3
18812024-02-15T02:57:53.639Z30/30 Write block 177 len 16384 data: 3 3 3 3
18822024-02-15T02:57:53.639Z----------------------------------------------------------------
18832024-02-15T02:57:53.639Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18842024-02-15T02:57:53.639ZStates: Active Active Active
18852024-02-15T02:57:53.639ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18862024-02-15T02:57:53.640Z 1 Acked 1000 Write 4 Done Done Done false
18872024-02-15T02:57:53.640Z 2 Acked 1001 Write 3 Sent Done Done false
18882024-02-15T02:57:53.640Z 3 Acked 1002 Read 7 Sent Done Done false
18892024-02-15T02:57:53.640Z 4 Acked 1003 Flush 0 Sent Done Done false
18902024-02-15T02:57:53.640Z 5 Acked 1004 Write 6 Sent Done Done false
18912024-02-15T02:57:53.640Z 6 Acked 1005 Write 7 Sent Done Done false
18922024-02-15T02:57:53.641Z 7 Acked 1006 Read 5 Sent Done Done false
18932024-02-15T02:57:53.641Z 8 Acked 1007 Flush 0 Sent Done Done false
18942024-02-15T02:57:53.641Z 9 Acked 1008 Write 7 Sent Done Done false
18952024-02-15T02:57:53.642Z 10 Acked 1009 Read 10 Sent Done Done false
18962024-02-15T02:57:53.642Z 11 Acked 1010 Read 10 Sent Done Done false
18972024-02-15T02:57:53.642Z 12 Acked 1011 Read 5 Sent Done Done false
18982024-02-15T02:57:53.642Z 13 Acked 1012 Write 7 Sent Done Done false
18992024-02-15T02:57:53.642Z 14 Acked 1013 Write 2 Sent Done Done false
19002024-02-15T02:57:53.643Z 15 Acked 1014 Write 2 Sent Done Done false
19012024-02-15T02:57:53.643Z 16 Acked 1015 Write 8 Sent Done Done false
19022024-02-15T02:57:53.643Z 17 Acked 1016 Read 6 Sent Done Done false
19032024-02-15T02:57:53.643Z 18 Acked 1017 Read 7 Sent Done Done false
19042024-02-15T02:57:53.643Z 19 Acked 1018 Read 8 Sent Done Done false
19052024-02-15T02:57:53.644Z 20 Acked 1019 Flush 0 Sent Done Done false
19062024-02-15T02:57:53.644Z 21 Acked 1020 Flush 0 Sent Done Done false
19072024-02-15T02:57:53.644Z 22 Acked 1021 Write 7 Sent Done Done false
19082024-02-15T02:57:53.644Z 23 Acked 1022 Read 5 Sent Done Done false
19092024-02-15T02:57:53.645Z 24 Acked 1023 Write 3 Sent Done Done false
19102024-02-15T02:57:53.645Z 25 Acked 1024 Flush 0 Sent Done Done false
19112024-02-15T02:57:53.645Z 26 Acked 1025 Flush 0 Sent Done Done false
19122024-02-15T02:57:53.645Z 27 Acked 1026 Write 4 Sent Done Done false
19132024-02-15T02:57:53.646Z 28 Acked 1027 Write 6 Sent Done Done false
19142024-02-15T02:57:53.646Z 29 Acked 1028 Write 8 Sent Sent Done false
19152024-02-15T02:57:53.646Z 30 Acked 1029 Write 4 Sent Sent Sent false
19162024-02-15T02:57:53.646Z STATES DS:0 DS:1 DS:2 TOTAL
19172024-02-15T02:57:53.647Z New 0 0 0 0
19182024-02-15T02:57:53.647Z Sent 29 2 1 32
19192024-02-15T02:57:53.647Z Done 1 28 29 58
19202024-02-15T02:57:53.647Z Skipped 0 0 0 0
19212024-02-15T02:57:53.647Z Error 0 0 0 0
19222024-02-15T02:57:53.648ZLast Flush: 0 1025 1025
19232024-02-15T02:57:53.648ZDownstairs last five completed:
19242024-02-15T02:57:53.648ZUpstairs last five completed: 30 29 28 27 26
19252024-02-15T02:57:53.648Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:52.586698185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1199}
19262024-02-15T02:57:53.649ZWrote out file "/tmp/test_repair_verify.data"
19272024-02-15T02:57:53.658Z
19282024-02-15T02:57:53.658ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
19292024-02-15T02:57:53.678ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19302024-02-15T02:57:53.679Z 0 000-019 14 14 14 25 25 25 F F F
19312024-02-15T02:57:53.679Z 1 020-039 11 16 16 19 31 31 F F F <---
19322024-02-15T02:57:53.679Z 2 040-059 8 16 16 14 31 31 F F F <---
19332024-02-15T02:57:53.679Z 3 060-079 11 11 11 19 19 19 F F F
19342024-02-15T02:57:53.679Z 4 080-099 14 16 16 27 31 31 F F F <---
19352024-02-15T02:57:53.679Z 5 100-119 14 14 14 25 25 25 F F F
19362024-02-15T02:57:53.679Z 6 120-139 14 16 16 25 33 33 F F F <---
19372024-02-15T02:57:53.679Z 7 140-159 14 14 14 27 27 27 F F F
19382024-02-15T02:57:53.679Z 8 160-179 14 14 14 25 25 25 F T T <---
19392024-02-15T02:57:53.679Z 9 180-199 11 11 11 19 19 19 F T T <---
19402024-02-15T02:57:53.679Z 10 200-219 15 15 15 28 28 28 F F F
19412024-02-15T02:57:53.679Z 11 220-239 11 16 16 19 31 31 T F F <---
19422024-02-15T02:57:53.679Z 12 240-259 15 15 15 28 28 28 F F F
19432024-02-15T02:57:53.679Z 13 260-279 15 15 15 28 28 28 F F F
19442024-02-15T02:57:53.679Z 14 280-299 1 16 16 1 31 31 F F F <---
19452024-02-15T02:57:53.679Z 15 300-319 12 16 16 20 33 33 F F F <---
19462024-02-15T02:57:53.679Z 16 320-339 9 9 9 15 15 15 F F F
19472024-02-15T02:57:53.679Z 17 340-359 9 9 9 15 15 15 F F F
19482024-02-15T02:57:53.680Z 18 360-379 15 15 15 28 28 28 F F F
19492024-02-15T02:57:53.680Z 19 380-399 15 15 15 28 28 28 F F F
19502024-02-15T02:57:53.680Z 20 400-419 12 16 16 23 30 30 F F F <---
19512024-02-15T02:57:53.680Z 21 420-439 9 9 9 15 15 15 F F F
19522024-02-15T02:57:53.680Z 22 440-459 12 12 12 22 22 22 F F F
19532024-02-15T02:57:53.680Z 23 460-479 15 15 15 28 28 28 F F F
19542024-02-15T02:57:53.680Z 24 480-499 11 11 11 19 19 19 F F F
19552024-02-15T02:57:53.680Z 25 500-519 14 16 16 25 30 30 F F F <---
19562024-02-15T02:57:53.680Z 26 520-539 4 4 4 7 7 7 F F F
19572024-02-15T02:57:53.680Z 27 540-559 11 16 16 19 29 29 F T T <---
19582024-02-15T02:57:53.680Z 28 560-579 12 12 12 22 22 22 F F F
19592024-02-15T02:57:53.680Z 29 580-599 13 13 13 24 24 24 F F F
19602024-02-15T02:57:53.680ZMax gen: 16, Max flush: 33
19612024-02-15T02:57:53.680ZError: Difference in extent metadata found!
19622024-02-15T02:57:53.680ZA Difference in extent metadata is expected here
19632024-02-15T02:57:53.680ZOn loop 8
19642024-02-15T02:57:53.680Z
19652024-02-15T02:57:53.680ZVerifying data now
19662024-02-15T02:57:58.263ZNo change after verify
19672024-02-15T02:57:58.263ZLoop: 8 Downstairs dump after verify (and repair):
19682024-02-15T02:57:58.281ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19692024-02-15T02:57:58.281Z 0 000-019 14 14 14 25 25 25 F F F
19702024-02-15T02:57:58.281Z 1 020-039 16 16 16 31 31 31 F F F
19712024-02-15T02:57:58.281Z 2 040-059 16 16 16 31 31 31 F F F
19722024-02-15T02:57:58.281Z 3 060-079 11 11 11 19 19 19 F F F
19732024-02-15T02:57:58.281Z 4 080-099 16 16 16 31 31 31 F F F
19742024-02-15T02:57:58.281Z 5 100-119 14 14 14 25 25 25 F F F
19752024-02-15T02:57:58.281Z 6 120-139 16 16 16 33 33 33 F F F
19762024-02-15T02:57:58.282Z 7 140-159 14 14 14 27 27 27 F F F
19772024-02-15T02:57:58.282Z 8 160-179 17 17 17 34 34 34 F F F
19782024-02-15T02:57:58.282Z 9 180-199 17 17 17 34 34 34 F F F
19792024-02-15T02:57:58.282Z 10 200-219 15 15 15 28 28 28 F F F
19802024-02-15T02:57:58.282Z 11 220-239 16 16 16 31 31 31 F F F
19812024-02-15T02:57:58.282Z 12 240-259 15 15 15 28 28 28 F F F
19822024-02-15T02:57:58.383Z 13 260-279 15 15 15 28 28 28 F F F
19832024-02-15T02:57:58.383Z 14 280-299 16 16 16 31 31 31 F F F
19842024-02-15T02:57:58.384Z 15 300-319 16 16 16 33 33 33 F F F
19852024-02-15T02:57:58.384Z 16 320-339 9 9 9 15 15 15 F F F
19862024-02-15T02:57:58.384Z 17 340-359 9 9 9 15 15 15 F F F
19872024-02-15T02:57:58.384Z 18 360-379 15 15 15 28 28 28 F F F
19882024-02-15T02:57:58.384Z 19 380-399 15 15 15 28 28 28 F F F
19892024-02-15T02:57:58.384Z 20 400-419 16 16 16 30 30 30 F F F
19902024-02-15T02:57:58.384Z 21 420-439 9 9 9 15 15 15 F F F
19912024-02-15T02:57:58.385Z 22 440-459 12 12 12 22 22 22 F F F
19922024-02-15T02:57:58.385Z 23 460-479 15 15 15 28 28 28 F F F
19932024-02-15T02:57:58.385Z 24 480-499 11 11 11 19 19 19 F F F
19942024-02-15T02:57:58.385Z 25 500-519 16 16 16 30 30 30 F F F
19952024-02-15T02:57:58.385Z 26 520-539 4 4 4 7 7 7 F F F
19962024-02-15T02:57:58.385Z 27 540-559 17 17 17 34 34 34 F F F
19972024-02-15T02:57:58.386Z 28 560-579 12 12 12 22 22 22 F F F
19982024-02-15T02:57:58.386Z 29 580-599 13 13 13 24 24 24 F F F
19992024-02-15T02:57:58.386ZMax gen: 17, Max flush: 34
20002024-02-15T02:57:58.386Z
20012024-02-15T02:57:58.386ZBegin loop 9
20022024-02-15T02:57:58.387ZDownstairs to restart: 1
20032024-02-15T02:57:58.387Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
20042024-02-15T02:57:58.387Z,"time":"2024-02-15T02:57:57.250114863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207}
20052024-02-15T02:57:58.387Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250434673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207}
20062024-02-15T02:57:58.387Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250483907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207}
20072024-02-15T02:57:58.387Z{"msg":"Crucible d3d090d6-c053-417f-b3dc-cb8c172df500 has session id: 07ed9f9a-3368-4cca-bacd-1d058da7ccfc","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250511573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20082024-02-15T02:57:58.388Z{"msg":"Upstairs opts: Upstairs UUID: d3d090d6-c053-417f-b3dc-cb8c172df500, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250564577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20092024-02-15T02:57:58.388Z{"msg":"Crucible stats registered with UUID: d3d090d6-c053-417f-b3dc-cb8c172df500","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250605942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20102024-02-15T02:57:58.388Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250633968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207}
20112024-02-15T02:57:58.388Z{"msg":"d3d090d6-c053-417f-b3dc-cb8c172df500 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250655245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20122024-02-15T02:57:58.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250678782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20132024-02-15T02:57:58.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250718987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20142024-02-15T02:57:58.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250772941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20152024-02-15T02:57:58.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250801287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20162024-02-15T02:57:58.390Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250827084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20172024-02-15T02:57:58.390Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250851931Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20182024-02-15T02:57:58.390Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250877887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20192024-02-15T02:57:58.390Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.250909073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20202024-02-15T02:57:58.390Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.25093561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20212024-02-15T02:57:58.391Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:57.250988543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20222024-02-15T02:57:58.391Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251055735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20232024-02-15T02:57:58.391Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:57.251089361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20242024-02-15T02:57:58.391Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251129056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20252024-02-15T02:57:58.392Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:57.251160482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20262024-02-15T02:57:58.392Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251197777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20272024-02-15T02:57:58.392Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251224404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20282024-02-15T02:57:58.393Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251263059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20292024-02-15T02:57:58.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251721192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20302024-02-15T02:57:58.393Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251802352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20312024-02-15T02:57:58.393Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251857725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20322024-02-15T02:57:58.393Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.251877932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20332024-02-15T02:57:58.393Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.252335175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20342024-02-15T02:57:58.394Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.25237357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20352024-02-15T02:57:58.394Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.253002481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20362024-02-15T02:57:58.394Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.253054915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20372024-02-15T02:57:58.394Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.25309105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20382024-02-15T02:57:58.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.253106928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20392024-02-15T02:57:58.395Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.253128186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20402024-02-15T02:57:58.395Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:57.253141614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20412024-02-15T02:57:59.302Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.252779222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20422024-02-15T02:57:59.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.252909556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20432024-02-15T02:57:59.314Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.253780387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20442024-02-15T02:57:59.314Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.253830581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20452024-02-15T02:57:59.314Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.254319889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20462024-02-15T02:57:59.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.254879929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20472024-02-15T02:57:59.314Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.254925293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20482024-02-15T02:57:59.314Z{"msg":"[0]R flush_numbers[0..12]: [25, 31, 31, 19, 31, 25, 33, 27, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.254956989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20492024-02-15T02:57:59.314Z{"msg":"[0]R generation[0..12]: [14, 16, 16, 11, 16, 14, 16, 14, 17, 17, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255000894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20502024-02-15T02:57:59.314Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.25503075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20512024-02-15T02:57:59.314Z{"msg":"[1]R flush_numbers[0..12]: [25, 31, 31, 19, 31, 25, 33, 27, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255057627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20522024-02-15T02:57:59.314Z{"msg":"[1]R generation[0..12]: [14, 16, 16, 11, 16, 14, 16, 14, 17, 17, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255076365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20532024-02-15T02:57:59.314Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255102601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20542024-02-15T02:57:59.315ZWait for a query_work_queue command to finish before sending IO
20552024-02-15T02:57:59.315Z{"msg":"[2]R flush_numbers[0..12]: [25, 31, 31, 19, 31, 25, 33, 27, 34, 34, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255136057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20562024-02-15T02:57:59.315Z{"msg":"[2]R generation[0..12]: [14, 16, 16, 11, 16, 14, 16, 14, 17, 17, 15, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255161874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20572024-02-15T02:57:59.315Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255180121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20582024-02-15T02:57:59.315Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255204618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20592024-02-15T02:57:59.315Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255223326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20602024-02-15T02:57:59.315Z{"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255247663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20612024-02-15T02:57:59.315Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255288898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20622024-02-15T02:57:59.315Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255309525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20632024-02-15T02:57:59.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255334092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20642024-02-15T02:57:59.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255374567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20652024-02-15T02:57:59.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255413082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20662024-02-15T02:57:59.315Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255437729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20672024-02-15T02:57:59.315Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255472765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20682024-02-15T02:57:59.315Z{"msg":"d3d090d6-c053-417f-b3dc-cb8c172df500 is now active with session: 07ed9f9a-3368-4cca-bacd-1d058da7ccfc","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255519209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20692024-02-15T02:57:59.315Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255537467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207,"session_id":"07ed9f9a-3368-4cca-bacd-1d058da7ccfc"}
20702024-02-15T02:57:59.315Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:57:58.255553305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207}
20712024-02-15T02:57:59.316ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20722024-02-15T02:57:59.316ZLoading write count information from file "/tmp/test_repair_verify.data"
20732024-02-15T02:57:59.316ZRun Repair workload
20742024-02-15T02:57:59.316Z01/30 Write block 183 len 16384 data: 2 3 3 3
20752024-02-15T02:57:59.316Z02/30 Write block 335 len 16384 data: 5 4 4 4
20762024-02-15T02:57:59.316Z03/30 Read block 136 len 8192
20772024-02-15T02:57:59.316Z04/30 Read block 453 len 16384
20782024-02-15T02:57:59.316Z05/30 Read block 431 len 36864
20792024-02-15T02:57:59.350Z06/30 Read block 324 len 4096
20802024-02-15T02:57:59.350Z07/30 Read block 235 len 40960
20812024-02-15T02:57:59.350Z08/30 Read block 236 len 32768
20822024-02-15T02:57:59.350Z09/30 Write block 376 len 8192 data: 3 3
20832024-02-15T02:57:59.351Z10/30 Read block 256 len 28672
20842024-02-15T02:57:59.351Z11/30 Write block 582 len 4096 data: 2
20852024-02-15T02:57:59.351Z12/30 Read block 110 len 40960
20862024-02-15T02:57:59.351Z13/30 Write block 225 len 24576 data: 2 4 4 4 4 3
20872024-02-15T02:57:59.351Z14/30 Read block 451 len 36864
20882024-02-15T02:57:59.351Z15/30 Write block 145 len 16384 data: 2 2 3 3
20892024-02-15T02:57:59.351Z16/30 Write block 575 len 8192 data: 3 2
20902024-02-15T02:57:59.351Z17/30 Write block 132 len 40960 data: 3 4 4 3 3 3 2 2 2 2
20912024-02-15T02:57:59.352Z18/30 Read block 577 len 16384
20922024-02-15T02:57:59.352Z19/30 Write block 490 len 4096 data: 3
20932024-02-15T02:57:59.352Z20/30 Read block 75 len 4096
20942024-02-15T02:57:59.352Z21/30 Flush
20952024-02-15T02:57:59.352Z22/30 Write block 156 len 32768 data: 3 3 3 3 3 3 3 3
20962024-02-15T02:57:59.352Z23/30 Read block 334 len 4096
20972024-02-15T02:57:59.352Z24/30 Write block 45 len 4096 data: 3
20982024-02-15T02:57:59.352Z25/30 Write block 563 len 24576 data: 4 4 4 2 2 2
20992024-02-15T02:57:59.353Z26/30 Write block 108 len 4096 data: 3
21002024-02-15T02:57:59.353Z27/30 Write block 162 len 8192 data: 4 4
21012024-02-15T02:57:59.353Z28/30 Write block 121 len 4096 data: 2
21022024-02-15T02:57:59.353Z29/30 Read block 498 len 32768
21032024-02-15T02:57:59.353Z30/30 Write block 30 len 20480 data: 3 3 3 3 3
21042024-02-15T02:57:59.353Z----------------------------------------------------------------
21052024-02-15T02:57:59.353Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
21062024-02-15T02:57:59.353ZStates: Active Active Active
21072024-02-15T02:57:59.353ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
21082024-02-15T02:57:59.353Z 1 Acked 1000 Write 4 Done Done Done false
21092024-02-15T02:57:59.354Z 2 Acked 1001 Write 4 Done Sent Done false
21102024-02-15T02:57:59.354Z 3 Acked 1002 Read 2 Done Sent Done false
21112024-02-15T02:57:59.354Z 4 Acked 1003 Read 4 Done Sent Done false
21122024-02-15T02:57:59.354Z 5 Acked 1004 Read 9 Done Sent Done false
21132024-02-15T02:57:59.354Z 6 Acked 1005 Read 1 Done Sent Done false
21142024-02-15T02:57:59.354Z 7 Acked 1006 Read 10 Done Sent Done false
21152024-02-15T02:57:59.354Z 8 Acked 1007 Read 8 Done Sent Done false
21162024-02-15T02:57:59.355Z 9 Acked 1008 Write 2 Done Sent Done false
21172024-02-15T02:57:59.355Z 10 Acked 1009 Read 7 Done Sent Done false
21182024-02-15T02:57:59.355Z 11 Acked 1010 Write 1 Done Sent Done false
21192024-02-15T02:57:59.355Z 12 Acked 1011 Read 10 Done Sent Done false
21202024-02-15T02:57:59.355Z 13 Acked 1012 Write 6 Done Sent Done false
21212024-02-15T02:57:59.355Z 14 Acked 1013 Read 9 Done Sent Done false
21222024-02-15T02:57:59.355Z 15 Acked 1014 Write 4 Done Sent Done false
21232024-02-15T02:57:59.356Z 16 Acked 1015 Write 2 Done Sent Done false
21242024-02-15T02:57:59.356Z 17 Acked 1016 Write 10 Done Sent Done false
21252024-02-15T02:57:59.356Z 18 Acked 1017 Read 4 Done Sent Done false
21262024-02-15T02:57:59.356Z 19 Acked 1018 Write 1 Done Sent Done false
21272024-02-15T02:57:59.356Z 20 Acked 1019 Read 1 Done Sent Done false
21282024-02-15T02:57:59.356Z 21 Acked 1020 Flush 0 Done Sent Done false
21292024-02-15T02:57:59.356Z 22 Acked 1021 Write 8 Done Sent Done false
21302024-02-15T02:57:59.356Z 23 Acked 1022 Read 1 Done Sent Done false
21312024-02-15T02:57:59.357Z 24 Acked 1023 Write 1 Done Sent Done false
21322024-02-15T02:57:59.357Z 25 Acked 1024 Write 6 Done Sent Done false
21332024-02-15T02:57:59.357Z 26 Acked 1025 Write 1 Done Sent Done false
21342024-02-15T02:57:59.357Z 27 Acked 1026 Write 2 Done Sent Done false
21352024-02-15T02:57:59.357Z 28 Acked 1027 Write 1 Done Sent Done false
21362024-02-15T02:57:59.357Z 29 Acked 1028 Read 8 Done Sent Done false
21372024-02-15T02:57:59.357Z 30 Acked 1029 Write 5 Sent Sent Sent false
21382024-02-15T02:57:59.357Z STATES DS:0 DS:1 DS:2 TOTAL
21392024-02-15T02:57:59.357Z New 0 0 0 0
21402024-02-15T02:57:59.357Z Sent 1 29 1 31
21412024-02-15T02:57:59.357Z Done 29 1 29 59
21422024-02-15T02:57:59.357Z Skipped 0 0 0 0
21432024-02-15T02:57:59.357Z Error 0 0 0 0
21442024-02-15T02:57:59.357ZLast Flush: 1020 0 1020
21452024-02-15T02:57:59.357ZDownstairs last five completed:
21462024-02-15T02:57:59.358ZUpstairs last five completed: 30 29 28 27 26
21472024-02-15T02:57:59.358Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:57:58.282620693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1207}
21482024-02-15T02:57:59.358ZWrote out file "/tmp/test_repair_verify.data"
21492024-02-15T02:57:59.358Z
21502024-02-15T02:57:59.382ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21512024-02-15T02:57:59.382ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21522024-02-15T02:57:59.382Z 0 000-019 14 14 14 25 25 25 F F F
21532024-02-15T02:57:59.382Z 1 020-039 16 16 16 31 31 31 T F T <---
21542024-02-15T02:57:59.382Z 2 040-059 16 16 16 31 31 31 T F T <---
21552024-02-15T02:57:59.382Z 3 060-079 11 11 11 19 19 19 F F F
21562024-02-15T02:57:59.382Z 4 080-099 16 16 16 31 31 31 F F F
21572024-02-15T02:57:59.382Z 5 100-119 14 14 14 25 25 25 T F T <---
21582024-02-15T02:57:59.382Z 6 120-139 18 16 18 35 33 35 T F T <---
21592024-02-15T02:57:59.383Z 7 140-159 18 14 18 35 27 35 T F T <---
21602024-02-15T02:57:59.383Z 8 160-179 17 17 17 34 34 34 T F T <---
21612024-02-15T02:57:59.383Z 9 180-199 18 17 18 35 34 35 F T F <---
21622024-02-15T02:57:59.383Z 10 200-219 15 15 15 28 28 28 F F F
21632024-02-15T02:57:59.383Z 11 220-239 18 16 18 35 31 35 F F F <---
21642024-02-15T02:57:59.383Z 12 240-259 15 15 15 28 28 28 F F F
21652024-02-15T02:57:59.383Z 13 260-279 15 15 15 28 28 28 F F F
21662024-02-15T02:57:59.383Z 14 280-299 16 16 16 31 31 31 F F F
21672024-02-15T02:57:59.383Z 15 300-319 16 16 16 33 33 33 F F F
21682024-02-15T02:57:59.383Z 16 320-339 18 9 18 35 15 35 F F F <---
21692024-02-15T02:57:59.383Z 17 340-359 9 9 9 15 15 15 F F F
21702024-02-15T02:57:59.383Z 18 360-379 18 15 18 35 28 35 F F F <---
21712024-02-15T02:57:59.383Z 19 380-399 15 15 15 28 28 28 F F F
21722024-02-15T02:57:59.384Z 20 400-419 16 16 16 30 30 30 F F F
21732024-02-15T02:57:59.384Z 21 420-439 9 9 9 15 15 15 F F F
21742024-02-15T02:57:59.384Z 22 440-459 12 12 12 22 22 22 F F F
21752024-02-15T02:57:59.384Z 23 460-479 15 15 15 28 28 28 F F F
21762024-02-15T02:57:59.384Z 24 480-499 18 11 18 35 19 35 F F F <---
21772024-02-15T02:57:59.384Z 25 500-519 16 16 16 30 30 30 F F F
21782024-02-15T02:57:59.384Z 26 520-539 4 4 4 7 7 7 F F F
21792024-02-15T02:57:59.384Z 27 540-559 17 17 17 34 34 34 F F F
21802024-02-15T02:57:59.384Z 28 560-579 18 12 18 35 22 35 T F T <---
21812024-02-15T02:57:59.384Z 29 580-599 18 13 18 35 24 35 F F F <---
21822024-02-15T02:57:59.384ZMax gen: 18, Max flush: 35
21832024-02-15T02:57:59.384ZError: Difference in extent metadata found!
21842024-02-15T02:57:59.384ZA Difference in extent metadata is expected here
21852024-02-15T02:57:59.385ZOn loop 9
21862024-02-15T02:57:59.385Z
21872024-02-15T02:57:59.389ZVerifying data now
21882024-02-15T02:58:05.011ZNo change after verify
21892024-02-15T02:58:05.012ZLoop: 9 Downstairs dump after verify (and repair):
21902024-02-15T02:58:05.035ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21912024-02-15T02:58:05.080Z 0 000-019 14 14 14 25 25 25 F F F
21922024-02-15T02:58:05.080Z 1 020-039 19 19 19 36 36 36 F F F
21932024-02-15T02:58:05.080Z 2 040-059 19 19 19 36 36 36 F F F
21942024-02-15T02:58:05.080Z 3 060-079 11 11 11 19 19 19 F F F
21952024-02-15T02:58:05.080Z 4 080-099 16 16 16 31 31 31 F F F
21962024-02-15T02:58:05.080Z 5 100-119 19 19 19 36 36 36 F F F
21972024-02-15T02:58:05.081Z 6 120-139 19 19 19 36 36 36 F F F
21982024-02-15T02:58:05.081Z 7 140-159 19 19 19 36 36 36 F F F
21992024-02-15T02:58:05.081Z 8 160-179 19 19 19 36 36 36 F F F
22002024-02-15T02:58:05.081Z 9 180-199 18 18 18 35 35 35 F F F
22012024-02-15T02:58:05.081Z 10 200-219 15 15 15 28 28 28 F F F
22022024-02-15T02:58:05.081Z 11 220-239 18 18 18 35 35 35 F F F
22032024-02-15T02:58:05.081Z 12 240-259 15 15 15 28 28 28 F F F
22042024-02-15T02:58:05.081Z 13 260-279 15 15 15 28 28 28 F F F
22052024-02-15T02:58:05.081Z 14 280-299 16 16 16 31 31 31 F F F
22062024-02-15T02:58:05.082Z 15 300-319 16 16 16 33 33 33 F F F
22072024-02-15T02:58:05.082Z 16 320-339 18 18 18 35 35 35 F F F
22082024-02-15T02:58:05.082Z 17 340-359 9 9 9 15 15 15 F F F
22092024-02-15T02:58:05.082Z 18 360-379 18 18 18 35 35 35 F F F
22102024-02-15T02:58:05.082Z 19 380-399 15 15 15 28 28 28 F F F
22112024-02-15T02:58:05.082Z 20 400-419 16 16 16 30 30 30 F F F
22122024-02-15T02:58:05.082Z 21 420-439 9 9 9 15 15 15 F F F
22132024-02-15T02:58:05.082Z 22 440-459 12 12 12 22 22 22 F F F
22142024-02-15T02:58:05.083Z 23 460-479 15 15 15 28 28 28 F F F
22152024-02-15T02:58:05.083Z 24 480-499 18 18 18 35 35 35 F F F
22162024-02-15T02:58:05.083Z 25 500-519 16 16 16 30 30 30 F F F
22172024-02-15T02:58:05.083Z 26 520-539 4 4 4 7 7 7 F F F
22182024-02-15T02:58:05.083Z 27 540-559 17 17 17 34 34 34 F F F
22192024-02-15T02:58:05.083Z 28 560-579 19 19 19 36 36 36 F F F
22202024-02-15T02:58:05.084Z 29 580-599 18 18 18 35 35 35 F F F
22212024-02-15T02:58:05.084ZMax gen: 19, Max flush: 36
22222024-02-15T02:58:05.084Z
22232024-02-15T02:58:05.084ZBegin loop 10
22242024-02-15T02:58:05.084ZDownstairs to restart: 1
22252024-02-15T02:58:05.110Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
22262024-02-15T02:58:05.111Z,"time":"2024-02-15T02:58:03.9993015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215}
22272024-02-15T02:58:05.111Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999612701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215}
22282024-02-15T02:58:05.111Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999644047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215}
22292024-02-15T02:58:05.112Z{"msg":"Crucible f7dcd4df-4eba-42ae-9315-feb149495872 has session id: 1444b6a2-27eb-4aab-a876-737f9495ae89","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999661775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22302024-02-15T02:58:05.112Z{"msg":"Upstairs opts: Upstairs UUID: f7dcd4df-4eba-42ae-9315-feb149495872, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.99969865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22312024-02-15T02:58:05.112Z{"msg":"Crucible stats registered with UUID: f7dcd4df-4eba-42ae-9315-feb149495872","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999718398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22322024-02-15T02:58:05.112Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999736336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215}
22332024-02-15T02:58:05.113Z{"msg":"f7dcd4df-4eba-42ae-9315-feb149495872 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999774251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22342024-02-15T02:58:05.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999829084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22352024-02-15T02:58:05.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999876698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22362024-02-15T02:58:05.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999922132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22372024-02-15T02:58:05.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:03.999976166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22382024-02-15T02:58:05.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000093421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22392024-02-15T02:58:05.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000162622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22402024-02-15T02:58:05.114Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000220635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22412024-02-15T02:58:05.114Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000251121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22422024-02-15T02:58:05.114Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000284507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22432024-02-15T02:58:05.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000310884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22442024-02-15T02:58:05.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000343889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22452024-02-15T02:58:05.115Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:04.000381235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22462024-02-15T02:58:05.115Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:04.000432158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22472024-02-15T02:58:05.116Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:04.000459625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22482024-02-15T02:58:05.116Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000479863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22492024-02-15T02:58:05.117Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000529286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22502024-02-15T02:58:05.117Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.000545474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22512024-02-15T02:58:05.117Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.001002767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22522024-02-15T02:58:05.117Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.001040692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22532024-02-15T02:58:05.117Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.001076408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22542024-02-15T02:58:05.118Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.001093196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22552024-02-15T02:58:05.118Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.00161924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22562024-02-15T02:58:05.118Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.001655765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22572024-02-15T02:58:05.118Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.002282167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22582024-02-15T02:58:05.119Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.00233466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22592024-02-15T02:58:05.119Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.002353968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22602024-02-15T02:58:05.119Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.002367246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22612024-02-15T02:58:05.119Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.002382084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22622024-02-15T02:58:05.120Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:04.002404451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22632024-02-15T02:58:06.056Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.001943052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22642024-02-15T02:58:06.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.002093453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22652024-02-15T02:58:06.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.002871606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22662024-02-15T02:58:06.056Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.002953085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22672024-02-15T02:58:06.056Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.003425776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22682024-02-15T02:58:06.056Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.003996405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22692024-02-15T02:58:06.057Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004025741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22702024-02-15T02:58:06.062Z{"msg":"[0]R flush_numbers[0..12]: [25, 36, 36, 19, 31, 36, 36, 36, 36, 35, 28, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004045728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22712024-02-15T02:58:06.063Z{"msg":"[0]R generation[0..12]: [14, 19, 19, 11, 16, 19, 19, 19, 19, 18, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004085563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22722024-02-15T02:58:06.063Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.00411588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22732024-02-15T02:58:06.063Z{"msg":"[1]R flush_numbers[0..12]: [25, 36, 36, 19, 31, 36, 36, 36, 36, 35, 28, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004132088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22742024-02-15T02:58:06.063Z{"msg":"[1]R generation[0..12]: [14, 19, 19, 11, 16, 19, 19, 19, 19, 18, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004153105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22752024-02-15T02:58:06.063Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004177862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22762024-02-15T02:58:06.063Z{"msg":"[2]R flush_numbers[0..12]: [25, 36, 36, 19, 31, 36, 36, 36, 36, 35, 28, 35]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
22772024-02-15T02:58:06.063Z,"time":"2024-02-15T02:58:05.004205019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22782024-02-15T02:58:06.063Z{"msg":"[2]R generation[0..12]: [14, 19, 19, 11, 16, 19, 19, 19, 19, 18, 15, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004234235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22792024-02-15T02:58:06.064Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004260852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22802024-02-15T02:58:06.064Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004285798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22812024-02-15T02:58:06.064Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004311015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22822024-02-15T02:58:06.064Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004336222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22832024-02-15T02:58:06.064Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.0043537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22842024-02-15T02:58:06.064Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004378807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22852024-02-15T02:58:06.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004395424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22862024-02-15T02:58:06.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004421031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22872024-02-15T02:58:06.065Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004446018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22882024-02-15T02:58:06.065Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004473555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22892024-02-15T02:58:06.065Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004490993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22902024-02-15T02:58:06.065Z{"msg":"f7dcd4df-4eba-42ae-9315-feb149495872 is now active with session: 1444b6a2-27eb-4aab-a876-737f9495ae89","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004515789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22912024-02-15T02:58:06.065Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004531867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215,"session_id":"1444b6a2-27eb-4aab-a876-737f9495ae89"}
22922024-02-15T02:58:06.065Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:05.004555855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215}
22932024-02-15T02:58:06.066ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22942024-02-15T02:58:06.066ZLoading write count information from file "/tmp/test_repair_verify.data"
22952024-02-15T02:58:06.066ZRun Repair workload
22962024-02-15T02:58:06.066Z01/30 Flush
22972024-02-15T02:58:06.066Z02/30 Write block 27 len 20480 data: 2 3 3 4 4
22982024-02-15T02:58:06.066Z03/30 Read block 137 len 28672
22992024-02-15T02:58:06.066Z04/30 Flush
23002024-02-15T02:58:06.066Z05/30 Write block 224 len 20480 data: 3 3 5 5 5
23012024-02-15T02:58:06.066Z06/30 Write block 532 len 4096 data: 3
23022024-02-15T02:58:06.067Z07/30 Read block 452 len 36864
23032024-02-15T02:58:06.072Z08/30 Read block 166 len 4096
23042024-02-15T02:58:06.072Z09/30 Write block 418 len 8192 data: 2 2
23052024-02-15T02:58:06.072Z10/30 Write block 443 len 4096 data: 4
23062024-02-15T02:58:06.072Z11/30 Read block 446 len 40960
23072024-02-15T02:58:06.072Z12/30 Read block 448 len 36864
23082024-02-15T02:58:06.072Z13/30 Read block 491 len 4096
23092024-02-15T02:58:06.072Z14/30 Write block 96 len 28672 data: 4 5 5 4 3 3 3
23102024-02-15T02:58:06.073Z15/30 Write block 472 len 12288 data: 3 3 3
23112024-02-15T02:58:06.073Z16/30 Read block 76 len 36864
23122024-02-15T02:58:06.073Z17/30 Read block 376 len 20480
23132024-02-15T02:58:06.073Z18/30 Write block 460 len 4096 data: 2
23142024-02-15T02:58:06.073Z19/30 Read block 377 len 36864
23152024-02-15T02:58:06.077Z20/30 Read block 258 len 8192
23162024-02-15T02:58:06.077Z21/30 Read block 14 len 36864
23172024-02-15T02:58:06.077Z22/30 Write block 535 len 40960 data: 3 3 3 3 3 3 3 5 4 3
23182024-02-15T02:58:06.077Z23/30 Read block 281 len 40960
23192024-02-15T02:58:06.077Z24/30 Read block 553 len 8192
23202024-02-15T02:58:06.077Z25/30 Write block 341 len 8192 data: 2 2
23212024-02-15T02:58:06.084Z26/30 Read block 0 len 20480
23222024-02-15T02:58:06.084Z27/30 Read block 24 len 24576
23232024-02-15T02:58:06.084Z28/30 Read block 538 len 28672
23242024-02-15T02:58:06.084Z29/30 Write block 576 len 28672 data: 3 2 2 2 2 2 3
23252024-02-15T02:58:06.084Z30/30 Read block 501 len 20480
23262024-02-15T02:58:06.084Z----------------------------------------------------------------
23272024-02-15T02:58:06.084Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:29
23282024-02-15T02:58:06.084ZStates: Active Active Active
23292024-02-15T02:58:06.084ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
23302024-02-15T02:58:06.084Z 2 Acked 1001 Write 5 Done Sent Done false
23312024-02-15T02:58:06.084Z 3 Acked 1002 Read 7 Done Sent Done false
23322024-02-15T02:58:06.084Z 4 Acked 1003 Flush 0 Done Sent Done false
23332024-02-15T02:58:06.085Z 5 Acked 1004 Write 5 Done Sent Done false
23342024-02-15T02:58:06.085Z 6 Acked 1005 Write 1 Done Sent Done false
23352024-02-15T02:58:06.085Z 7 Acked 1006 Read 9 Done Sent Done false
23362024-02-15T02:58:06.085Z 8 Acked 1007 Read 1 Done Sent Done false
23372024-02-15T02:58:06.085Z 9 Acked 1008 Write 2 Done Sent Done false
23382024-02-15T02:58:06.085Z 10 Acked 1009 Write 1 Done Sent Done false
23392024-02-15T02:58:06.085Z 11 Acked 1010 Read 10 Done Sent Done false
23402024-02-15T02:58:06.085Z 12 Acked 1011 Read 9 Done Sent Done false
23412024-02-15T02:58:06.085Z 13 Acked 1012 Read 1 Done Sent Done false
23422024-02-15T02:58:06.085Z 14 Acked 1013 Write 7 Done Sent Done false
23432024-02-15T02:58:06.085Z 15 Acked 1014 Write 3 Done Sent Done false
23442024-02-15T02:58:06.085Z 16 Acked 1015 Read 9 Done Sent Done false
23452024-02-15T02:58:06.085Z 17 Acked 1016 Read 5 Done Sent Done false
23462024-02-15T02:58:06.085Z 18 Acked 1017 Write 1 Done Sent Done false
23472024-02-15T02:58:06.085Z 19 Acked 1018 Read 9 Done Sent Done false
23482024-02-15T02:58:06.086Z 20 Acked 1019 Read 2 Done Sent Done false
23492024-02-15T02:58:06.086Z 21 Acked 1020 Read 9 Done Sent Done false
23502024-02-15T02:58:06.086Z 22 Acked 1021 Write 10 Done Sent Done false
23512024-02-15T02:58:06.086Z 23 Acked 1022 Read 10 Done Sent Done false
23522024-02-15T02:58:06.086Z 24 Acked 1023 Read 2 Done Sent Done false
23532024-02-15T02:58:06.086Z 25 Acked 1024 Write 2 Done Sent Done false
23542024-02-15T02:58:06.086Z 26 Acked 1025 Read 5 Done Sent Done false
23552024-02-15T02:58:06.086Z 27 Acked 1026 Read 6 Done Sent Done false
23562024-02-15T02:58:06.086Z 28 Acked 1027 Read 7 Done Sent Done false
23572024-02-15T02:58:06.086Z 29 Acked 1028 Write 7 Done Sent Done false
23582024-02-15T02:58:06.086Z 30 Acked 1029 Read 5 Done Sent Sent false
23592024-02-15T02:58:06.086Z STATES DS:0 DS:1 DS:2 TOTAL
23602024-02-15T02:58:06.086Z New 0 0 0 0
23612024-02-15T02:58:06.086Z Sent 0 29 1 30
23622024-02-15T02:58:06.086Z Done 29 0 28 57
23632024-02-15T02:58:06.086Z Skipped 0 0 0 0
23642024-02-15T02:58:06.086Z Error 0 0 0 0
23652024-02-15T02:58:06.086ZLast Flush: 1003 1000 1003
23662024-02-15T02:58:06.086ZDownstairs last five completed: 1000
23672024-02-15T02:58:06.086ZUpstairs last five completed: 30 29 28 27 26
23682024-02-15T02:58:06.087Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:05.033342448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1215}
23692024-02-15T02:58:06.093ZWrote out file "/tmp/test_repair_verify.data"
23702024-02-15T02:58:06.093Z
23712024-02-15T02:58:06.099ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23722024-02-15T02:58:06.117ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23732024-02-15T02:58:06.117Z 0 000-019 14 14 14 25 25 25 F F F
23742024-02-15T02:58:06.117Z 1 020-039 20 19 20 38 36 38 F F F <---
23752024-02-15T02:58:06.118Z 2 040-059 19 19 19 36 36 36 F F F
23762024-02-15T02:58:06.118Z 3 060-079 11 11 11 19 19 19 F F F
23772024-02-15T02:58:06.118Z 4 080-099 16 16 16 31 31 31 T F T <---
23782024-02-15T02:58:06.118Z 5 100-119 19 19 19 36 36 36 T F T <---
23792024-02-15T02:58:06.118Z 6 120-139 19 19 19 36 36 36 F F F
23802024-02-15T02:58:06.118Z 7 140-159 19 19 19 36 36 36 F F F
23812024-02-15T02:58:06.130Z 8 160-179 19 19 19 36 36 36 F F F
23822024-02-15T02:58:06.130Z 9 180-199 18 18 18 35 35 35 F F F
23832024-02-15T02:58:06.131Z 10 200-219 15 15 15 28 28 28 F F F
23842024-02-15T02:58:06.131Z 11 220-239 18 18 18 35 35 35 T F T <---
23852024-02-15T02:58:06.131Z 12 240-259 15 15 15 28 28 28 F F F
23862024-02-15T02:58:06.131Z 13 260-279 15 15 15 28 28 28 F F F
23872024-02-15T02:58:06.131Z 14 280-299 16 16 16 31 31 31 F F F
23882024-02-15T02:58:06.131Z 15 300-319 16 16 16 33 33 33 F F F
23892024-02-15T02:58:06.131Z 16 320-339 18 18 18 35 35 35 F F F
23902024-02-15T02:58:06.131Z 17 340-359 9 9 9 15 15 15 T F T <---
23912024-02-15T02:58:06.131Z 18 360-379 18 18 18 35 35 35 F F F
23922024-02-15T02:58:06.131Z 19 380-399 15 15 15 28 28 28 F F F
23932024-02-15T02:58:06.132Z 20 400-419 16 16 16 30 30 30 T F T <---
23942024-02-15T02:58:06.132Z 21 420-439 9 9 9 15 15 15 F F F
23952024-02-15T02:58:06.132Z 22 440-459 12 12 12 22 22 22 T F T <---
23962024-02-15T02:58:06.132Z 23 460-479 15 15 15 28 28 28 T F T <---
23972024-02-15T02:58:06.132Z 24 480-499 18 18 18 35 35 35 F F F
23982024-02-15T02:58:06.132Z 25 500-519 16 16 16 30 30 30 F F F
23992024-02-15T02:58:06.132Z 26 520-539 4 4 4 7 7 7 T F T <---
24002024-02-15T02:58:06.132Z 27 540-559 17 17 17 34 34 34 T F T <---
24012024-02-15T02:58:06.132Z 28 560-579 19 19 19 36 36 36 T F T <---
24022024-02-15T02:58:06.132Z 29 580-599 18 18 18 35 35 35 T F T <---
24032024-02-15T02:58:06.133ZMax gen: 20, Max flush: 38
24042024-02-15T02:58:06.133ZError: Difference in extent metadata found!
24052024-02-15T02:58:06.133ZA Difference in extent metadata is expected here
24062024-02-15T02:58:06.133ZOn loop 10
24072024-02-15T02:58:06.133Z
24082024-02-15T02:58:06.133ZVerifying data now
24092024-02-15T02:58:10.743ZNo change after verify
24102024-02-15T02:58:10.743ZLoop: 10 Downstairs dump after verify (and repair):
24112024-02-15T02:58:10.759ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
24122024-02-15T02:58:10.759Z 0 000-019 14 14 14 25 25 25 F F F
24132024-02-15T02:58:10.759Z 1 020-039 20 20 20 38 38 38 F F F
24142024-02-15T02:58:10.760Z 2 040-059 19 19 19 36 36 36 F F F
24152024-02-15T02:58:10.760Z 3 060-079 11 11 11 19 19 19 F F F
24162024-02-15T02:58:10.760Z 4 080-099 21 21 21 39 39 39 F F F
24172024-02-15T02:58:10.760Z 5 100-119 21 21 21 39 39 39 F F F
24182024-02-15T02:58:10.760Z 6 120-139 19 19 19 36 36 36 F F F
24192024-02-15T02:58:10.760Z 7 140-159 19 19 19 36 36 36 F F F
24202024-02-15T02:58:10.760Z 8 160-179 19 19 19 36 36 36 F F F
24212024-02-15T02:58:10.760Z 9 180-199 18 18 18 35 35 35 F F F
24222024-02-15T02:58:10.760Z 10 200-219 15 15 15 28 28 28 F F F
24232024-02-15T02:58:10.777Z 11 220-239 21 21 21 39 39 39 F F F
24242024-02-15T02:58:10.777Z 12 240-259 15 15 15 28 28 28 F F F
24252024-02-15T02:58:10.777Z 13 260-279 15 15 15 28 28 28 F F F
24262024-02-15T02:58:10.777Z 14 280-299 16 16 16 31 31 31 F F F
24272024-02-15T02:58:10.777Z 15 300-319 16 16 16 33 33 33 F F F
24282024-02-15T02:58:10.777Z 16 320-339 18 18 18 35 35 35 F F F
24292024-02-15T02:58:10.777Z 17 340-359 21 21 21 39 39 39 F F F
24302024-02-15T02:58:10.777Z 18 360-379 18 18 18 35 35 35 F F F
24312024-02-15T02:58:10.777Z 19 380-399 15 15 15 28 28 28 F F F
24322024-02-15T02:58:10.777Z 20 400-419 21 21 21 39 39 39 F F F
24332024-02-15T02:58:10.777Z 21 420-439 9 9 9 15 15 15 F F F
24342024-02-15T02:58:10.778Z 22 440-459 21 21 21 39 39 39 F F F
24352024-02-15T02:58:10.778Z 23 460-479 21 21 21 39 39 39 F F F
24362024-02-15T02:58:10.778Z 24 480-499 18 18 18 35 35 35 F F F
24372024-02-15T02:58:10.778Z 25 500-519 16 16 16 30 30 30 F F F
24382024-02-15T02:58:10.778Z 26 520-539 21 21 21 39 39 39 F F F
24392024-02-15T02:58:10.778Z 27 540-559 21 21 21 39 39 39 F F F
24402024-02-15T02:58:10.778Z 28 560-579 21 21 21 39 39 39 F F F
24412024-02-15T02:58:10.778Z 29 580-599 21 21 21 39 39 39 F F F
24422024-02-15T02:58:10.778ZMax gen: 21, Max flush: 39
24432024-02-15T02:58:10.779Z
24442024-02-15T02:58:10.779ZBegin loop 11
24452024-02-15T02:58:10.779ZDownstairs to restart: 2
24462024-02-15T02:58:11.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727009815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223}
24472024-02-15T02:58:11.205Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727317296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223}
24482024-02-15T02:58:11.205ZCrucible runtime is spawned
24492024-02-15T02:58:11.205Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.72737095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223}
24502024-02-15T02:58:11.205Z{"msg":"Crucible d213d30f-df5c-4345-a752-af3cf31e3a98 has session id: 5ffc5929-b705-413a-a824-3cfe4ac106d0","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727430802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24512024-02-15T02:58:11.205Z{"msg":"Upstairs opts: Upstairs UUID: d213d30f-df5c-4345-a752-af3cf31e3a98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727491915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24522024-02-15T02:58:11.205Z{"msg":"Crucible stats registered with UUID: d213d30f-df5c-4345-a752-af3cf31e3a98","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727533249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24532024-02-15T02:58:11.205Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727571745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223}
24542024-02-15T02:58:11.205Z{"msg":"d213d30f-df5c-4345-a752-af3cf31e3a98 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727596952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24552024-02-15T02:58:11.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727622978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24562024-02-15T02:58:11.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727659034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24572024-02-15T02:58:11.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.72768397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24582024-02-15T02:58:11.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727724385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24592024-02-15T02:58:11.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727743663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24602024-02-15T02:58:11.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727776249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24612024-02-15T02:58:11.206Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727800386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24622024-02-15T02:58:11.206Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727842191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24632024-02-15T02:58:11.206Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727868838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24642024-02-15T02:58:11.206Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:09.727907762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24652024-02-15T02:58:11.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.727975854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24662024-02-15T02:58:11.206Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:09.72800754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24672024-02-15T02:58:11.206Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:09.728036117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24682024-02-15T02:58:11.206Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.728058574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24692024-02-15T02:58:11.206Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.728094559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24702024-02-15T02:58:11.207Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.728123226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24712024-02-15T02:58:11.207Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.728155391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24722024-02-15T02:58:11.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.728673866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24732024-02-15T02:58:11.207Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.72872565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24742024-02-15T02:58:11.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.728754996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24752024-02-15T02:58:11.207Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.728790842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24762024-02-15T02:58:11.207Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.729392427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24772024-02-15T02:58:11.207Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.729536529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24782024-02-15T02:58:11.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.729907222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24792024-02-15T02:58:11.207Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.729948257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24802024-02-15T02:58:11.207Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.729974554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24812024-02-15T02:58:11.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.73000443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24822024-02-15T02:58:11.207Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.730028947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24832024-02-15T02:58:11.207Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:09.730056263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24842024-02-15T02:58:11.783Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.729827325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24852024-02-15T02:58:11.783Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.729918944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24862024-02-15T02:58:11.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.730763648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24872024-02-15T02:58:11.801Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.730810832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24882024-02-15T02:58:11.801Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.73129938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24892024-02-15T02:58:11.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.731891376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24902024-02-15T02:58:11.801Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.731934361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24912024-02-15T02:58:11.802Z{"msg":"[0]R flush_numbers[0..12]: [25, 38, 36, 19, 39, 39, 36, 36, 36, 35, 28, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.731951799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24922024-02-15T02:58:11.802Z{"msg":"[0]R generation[0..12]: [14, 20, 19, 11, 21, 21, 19, 19, 19, 18, 15, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.731967607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24932024-02-15T02:58:11.802Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.731990984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24942024-02-15T02:58:11.802Z{"msg":"[1]R flush_numbers[0..12]: [25, 38, 36, 19, 39, 39, 36, 36, 36, 35, 28, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.7320218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24952024-02-15T02:58:11.802Z{"msg":"[1]R generation[0..12]: [14, 20, 19, 11, 21, 21, 19, 19, 19, 18, 15, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732045267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24962024-02-15T02:58:11.802Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732060535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24972024-02-15T02:58:11.802Z{"msg":"[2]R flush_numbers[0..12]: [25, 38, 36, 19, 39, 39, 36, 36, 36, 35, 28, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732075053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24982024-02-15T02:58:11.803Z{"msg":"[2]R generation[0..12]: [14, 20, 19, 11, 21, 21, 19, 19, 19, 18, 15, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732089361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
24992024-02-15T02:58:11.803Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.73210367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25002024-02-15T02:58:11.803Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732125077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25012024-02-15T02:58:11.803Z{"msg":"Generation requested: 22 >= found:22","v":Wait for a query_work_queue command to finish before sending IO
25022024-02-15T02:58:11.803Z0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732151334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25032024-02-15T02:58:11.803Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732175391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25042024-02-15T02:58:11.803Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732206097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25052024-02-15T02:58:11.803Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732229824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25062024-02-15T02:58:11.804Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.73225943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25072024-02-15T02:58:11.804Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732279888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25082024-02-15T02:58:11.804Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732304605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25092024-02-15T02:58:11.804Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732330611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25102024-02-15T02:58:11.804Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732346339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25112024-02-15T02:58:11.804Z{"msg":"d213d30f-df5c-4345-a752-af3cf31e3a98 is now active with session: 5ffc5929-b705-413a-a824-3cfe4ac106d0","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732359638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25122024-02-15T02:58:11.804Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732378285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223,"session_id":"5ffc5929-b705-413a-a824-3cfe4ac106d0"}
25132024-02-15T02:58:11.805Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:10.732401162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223}
25142024-02-15T02:58:11.805ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
25152024-02-15T02:58:11.805ZLoading write count information from file "/tmp/test_repair_verify.data"
25162024-02-15T02:58:11.805ZRun Repair workload
25172024-02-15T02:58:11.805Z01/30 Write block 44 len 16384 data: 3 4 4 4
25182024-02-15T02:58:11.805Z02/30 Read block 429 len 32768
25192024-02-15T02:58:11.805Z03/30 Write block 500 len 36864 data: 2 3 3 3 3 3 3 2 2
25202024-02-15T02:58:11.805Z04/30 Read block 191 len 28672
25212024-02-15T02:58:11.805Z05/30 Read block 102 len 32768
25222024-02-15T02:58:11.805Z06/30 Write block 217 len 16384 data: 3 3 4 5
25232024-02-15T02:58:11.805Z07/30 Write block 135 len 36864 data: 4 4 4 3 3 3 3 2 2
25242024-02-15T02:58:11.814Z08/30 Write block 30 len 40960 data: 5 5 4 4 4 5 3 3 4 3
25252024-02-15T02:58:11.814Z09/30 Flush
25262024-02-15T02:58:11.814Z10/30 Write block 64 len 24576 data: 4 4 3 2 3 3
25272024-02-15T02:58:11.814Z11/30 Write block 489 len 36864 data: 3 4 4 3 3 3 3 3 3
25282024-02-15T02:58:11.814Z12/30 Write block 286 len 36864 data: 2 2 2 2 2 2 2 2 2
25292024-02-15T02:58:11.814Z13/30 Read block 0 len 20480
25302024-02-15T02:58:11.814Z14/30 Write block 594 len 20480 data: 5 3 2 2 2
25312024-02-15T02:58:11.814Z15/30 Read block 117 len 40960
25322024-02-15T02:58:11.814Z16/30 Write block 477 len 12288 data: 2 3 3
25332024-02-15T02:58:11.815Z17/30 Flush
25342024-02-15T02:58:11.815Z18/30 Write block 62 len 4096 data: 3
25352024-02-15T02:58:11.815Z19/30 Flush
25362024-02-15T02:58:11.815Z20/30 Write block 526 len 28672 data: 2 2 2 2 2 2 4
25372024-02-15T02:58:11.815Z21/30 Write block 101 len 28672 data: 4 4 3 4 4 3 3
25382024-02-15T02:58:11.815Z22/30 Read block 71 len 32768
25392024-02-15T02:58:11.815Z23/30 Read block 495 len 28672
25402024-02-15T02:58:11.815Z24/30 Flush
25412024-02-15T02:58:11.815Z25/30 Flush
25422024-02-15T02:58:11.833Z26/30 Write block 205 len 16384 data: 2 3 3 3
25432024-02-15T02:58:11.833Z27/30 Write block 424 len 28672 data: 2 2 3 4 4 4 4
25442024-02-15T02:58:11.833Z28/30 Read block 471 len 12288
25452024-02-15T02:58:11.833Z29/30 Read block 177 len 16384
25462024-02-15T02:58:11.834Z30/30 Read block 274 len 4096
25472024-02-15T02:58:11.834Z----------------------------------------------------------------
25482024-02-15T02:58:11.834Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
25492024-02-15T02:58:11.834ZStates: Active Active Active
25502024-02-15T02:58:11.834ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
25512024-02-15T02:58:11.834Z 1 Acked 1000 Write 4 Done Done Done false
25522024-02-15T02:58:11.834Z 2 Acked 1001 Read 8 Done Done Done false
25532024-02-15T02:58:11.835Z 3 Acked 1002 Write 9 Done Done Done false
25542024-02-15T02:58:11.835Z 4 Acked 1003 Read 7 Done Done Done false
25552024-02-15T02:58:11.835Z 5 Acked 1004 Read 8 Done Done Sent false
25562024-02-15T02:58:11.835Z 6 Acked 1005 Write 4 Done Done Sent false
25572024-02-15T02:58:11.835Z 7 Acked 1006 Write 9 Done Done Sent false
25582024-02-15T02:58:11.835Z 8 Acked 1007 Write 10 Done Done Sent false
25592024-02-15T02:58:11.835Z 9 Acked 1008 Flush 0 Done Done Sent false
25602024-02-15T02:58:11.836Z 10 Acked 1009 Write 6 Done Done Sent false
25612024-02-15T02:58:11.836Z 11 Acked 1010 Write 9 Done Done Sent false
25622024-02-15T02:58:11.837Z 12 Acked 1011 Write 9 Done Done Sent false
25632024-02-15T02:58:11.837Z 13 Acked 1012 Read 5 Done Done Sent false
25642024-02-15T02:58:11.837Z 14 Acked 1013 Write 5 Done Done Sent false
25652024-02-15T02:58:11.837Z 15 Acked 1014 Read 10 Done Done Sent false
25662024-02-15T02:58:11.837Z 16 Acked 1015 Write 3 Done Done Sent false
25672024-02-15T02:58:11.838Z 17 Acked 1016 Flush 0 Done Done Sent false
25682024-02-15T02:58:11.838Z 18 Acked 1017 Write 1 Done Done Sent false
25692024-02-15T02:58:11.838Z 19 Acked 1018 Flush 0 Done Done Sent false
25702024-02-15T02:58:11.838Z 20 Acked 1019 Write 7 Done Done Sent false
25712024-02-15T02:58:11.839Z 21 Acked 1020 Write 7 Done Done Sent false
25722024-02-15T02:58:11.839Z 22 Acked 1021 Read 8 Done Done Sent false
25732024-02-15T02:58:11.839Z 23 Acked 1022 Read 7 Done Done Sent false
25742024-02-15T02:58:11.839Z 24 Acked 1023 Flush 0 Done Done Sent false
25752024-02-15T02:58:11.839Z 25 Acked 1024 Flush 0 Done Done Sent false
25762024-02-15T02:58:11.839Z 26 Acked 1025 Write 4 Done Done Sent false
25772024-02-15T02:58:11.839Z 27 Acked 1026 Write 7 Done Done Sent false
25782024-02-15T02:58:11.840Z 28 Acked 1027 Read 3 Done Done Sent false
25792024-02-15T02:58:11.840Z 29 Acked 1028 Read 4 Done Done Sent false
25802024-02-15T02:58:11.840Z 30 Acked 1029 Read 1 Done Sent Sent false
25812024-02-15T02:58:11.840Z STATES DS:0 DS:1 DS:2 TOTAL
25822024-02-15T02:58:11.840Z New 0 0 0 0
25832024-02-15T02:58:11.840Z Sent 0 1 26 27
25842024-02-15T02:58:11.841Z Done 30 29 4 63
25852024-02-15T02:58:11.841Z Skipped 0 0 0 0
25862024-02-15T02:58:11.841Z Error 0 0 0 0
25872024-02-15T02:58:11.841ZLast Flush: 1024 1024 0
25882024-02-15T02:58:11.841ZDownstairs last five completed:
25892024-02-15T02:58:11.842ZUpstairs last five completed: 30 29 28 27 26
25902024-02-15T02:58:11.842Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:10.766619255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1223}
25912024-02-15T02:58:11.842ZWrote out file "/tmp/test_repair_verify.data"
25922024-02-15T02:58:11.961Z
25932024-02-15T02:58:11.962ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25942024-02-15T02:58:11.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25952024-02-15T02:58:11.962Z 0 000-019 14 14 14 25 25 25 F F F
25962024-02-15T02:58:11.962Z 1 020-039 22 22 20 40 40 38 F F F <---
25972024-02-15T02:58:11.962Z 2 040-059 22 22 19 40 40 36 F F T <---
25982024-02-15T02:58:11.963Z 3 060-079 22 22 11 42 42 19 F F F <---
25992024-02-15T02:58:11.963Z 4 080-099 21 21 21 39 39 39 F F F
26002024-02-15T02:58:11.963Z 5 100-119 22 22 21 43 43 39 F F F <---
26012024-02-15T02:58:11.963Z 6 120-139 22 22 19 40 40 36 F F F <---
26022024-02-15T02:58:11.963Z 7 140-159 22 22 19 40 40 36 F F F <---
26032024-02-15T02:58:11.963Z 8 160-179 19 19 19 36 36 36 F F F
26042024-02-15T02:58:11.963Z 9 180-199 18 18 18 35 35 35 F F F
26052024-02-15T02:58:11.964Z 10 200-219 22 22 15 40 40 28 T T F <---
26062024-02-15T02:58:11.964Z 11 220-239 22 22 21 40 40 39 F F F <---
26072024-02-15T02:58:11.964Z 12 240-259 15 15 15 28 28 28 F F F
26082024-02-15T02:58:11.964Z 13 260-279 15 15 15 28 28 28 F F F
26092024-02-15T02:58:11.964Z 14 280-299 22 22 16 41 41 31 F F F <---
26102024-02-15T02:58:11.964Z 15 300-319 16 16 16 33 33 33 F F F
26112024-02-15T02:58:11.964Z 16 320-339 18 18 18 35 35 35 F F F
26122024-02-15T02:58:11.964Z 17 340-359 21 21 21 39 39 39 F F F
26132024-02-15T02:58:11.965Z 18 360-379 18 18 18 35 35 35 F F F
26142024-02-15T02:58:11.965Z 19 380-399 15 15 15 28 28 28 F F F
26152024-02-15T02:58:11.965Z 20 400-419 21 21 21 39 39 39 F F F
26162024-02-15T02:58:11.965Z 21 420-439 9 9 9 15 15 15 T T F <---
26172024-02-15T02:58:11.974Z 22 440-459 21 21 21 39 39 39 F F F
26182024-02-15T02:58:11.975Z 23 460-479 22 22 21 41 41 39 F F F <---
26192024-02-15T02:58:11.975Z 24 480-499 22 22 18 41 41 35 F F F <---
26202024-02-15T02:58:11.975Z 25 500-519 22 22 16 40 40 30 F F T <---
26212024-02-15T02:58:11.975Z 26 520-539 22 22 21 43 43 39 F F F <---
26222024-02-15T02:58:11.975Z 27 540-559 21 21 21 39 39 39 F F F
26232024-02-15T02:58:11.975Z 28 560-579 21 21 21 39 39 39 F F F
26242024-02-15T02:58:11.975Z 29 580-599 22 22 21 41 41 39 F F F <---
26252024-02-15T02:58:11.975ZMax gen: 22, Max flush: 43
26262024-02-15T02:58:11.976ZError: Difference in extent metadata found!
26272024-02-15T02:58:11.976ZA Difference in extent metadata is expected here
26282024-02-15T02:58:11.976ZOn loop 11
26292024-02-15T02:58:11.976Z
26302024-02-15T02:58:11.976ZVerifying data now
26312024-02-15T02:58:17.589ZNo change after verify
26322024-02-15T02:58:17.589ZLoop: 11 Downstairs dump after verify (and repair):
26332024-02-15T02:58:17.594ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
26342024-02-15T02:58:17.594Z 0 000-019 14 14 14 25 25 25 F F F
26352024-02-15T02:58:17.594Z 1 020-039 22 22 22 40 40 40 F F F
26362024-02-15T02:58:17.594Z 2 040-059 22 22 22 40 40 40 F F F
26372024-02-15T02:58:17.594Z 3 060-079 22 22 22 42 42 42 F F F
26382024-02-15T02:58:17.594Z 4 080-099 21 21 21 39 39 39 F F F
26392024-02-15T02:58:17.594Z 5 100-119 22 22 22 43 43 43 F F F
26402024-02-15T02:58:17.595Z 6 120-139 22 22 22 40 40 40 F F F
26412024-02-15T02:58:17.595Z 7 140-159 22 22 22 40 40 40 F F F
26422024-02-15T02:58:17.595Z 8 160-179 19 19 19 36 36 36 F F F
26432024-02-15T02:58:17.595Z 9 180-199 18 18 18 35 35 35 F F F
26442024-02-15T02:58:17.595Z 10 200-219 23 23 23 44 44 44 F F F
26452024-02-15T02:58:17.595Z 11 220-239 22 22 22 40 40 40 F F F
26462024-02-15T02:58:17.595Z 12 240-259 15 15 15 28 28 28 F F F
26472024-02-15T02:58:17.595Z 13 260-279 15 15 15 28 28 28 F F F
26482024-02-15T02:58:17.595Z 14 280-299 22 22 22 41 41 41 F F F
26492024-02-15T02:58:17.595Z 15 300-319 16 16 16 33 33 33 F F F
26502024-02-15T02:58:17.595Z 16 320-339 18 18 18 35 35 35 F F F
26512024-02-15T02:58:17.595Z 17 340-359 21 21 21 39 39 39 F F F
26522024-02-15T02:58:17.595Z 18 360-379 18 18 18 35 35 35 F F F
26532024-02-15T02:58:17.595Z 19 380-399 15 15 15 28 28 28 F F F
26542024-02-15T02:58:17.595Z 20 400-419 21 21 21 39 39 39 F F F
26552024-02-15T02:58:17.596Z 21 420-439 23 23 23 44 44 44 F F F
26562024-02-15T02:58:17.596Z 22 440-459 21 21 21 39 39 39 F F F
26572024-02-15T02:58:17.596Z 23 460-479 22 22 22 41 41 41 F F F
26582024-02-15T02:58:17.596Z 24 480-499 22 22 22 41 41 41 F F F
26592024-02-15T02:58:17.596Z 25 500-519 22 22 22 40 40 40 F F F
26602024-02-15T02:58:17.596Z 26 520-539 22 22 22 43 43 43 F F F
26612024-02-15T02:58:17.596Z 27 540-559 21 21 21 39 39 39 F F F
26622024-02-15T02:58:17.596Z 28 560-579 21 21 21 39 39 39 F F F
26632024-02-15T02:58:17.596Z 29 580-599 22 22 22 41 41 41 F F F
26642024-02-15T02:58:17.596ZMax gen: 23, Max flush: 44
26652024-02-15T02:58:17.596Z
26662024-02-15T02:58:17.596ZBegin loop 12
26672024-02-15T02:58:17.596ZDownstairs to restart: 2
26682024-02-15T02:58:17.619Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.563891165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231}
26692024-02-15T02:58:17.619Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564242801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231}
26702024-02-15T02:58:17.619Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564271927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231}
26712024-02-15T02:58:17.619Z{"msg":"Crucible a61aeae6-218f-4950-a7ec-d69aa07c502a has session id: 09d9cab4-fd25-4fd2-a92b-3d78788346e8","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564300034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26722024-02-15T02:58:17.664Z{"msg":"Upstairs opts: Upstairs UUID: a61aeae6-218f-4950-a7ec-d69aa07c502a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564366575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26732024-02-15T02:58:17.664Z{"msg":"Crucible stats registered with UUID: a61aeae6-218f-4950-a7ec-d69aa07c502a","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564400251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26742024-02-15T02:58:17.664ZCrucible runtime is spawned
26752024-02-15T02:58:17.664Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564554372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231}
26762024-02-15T02:58:17.665Z{"msg":"a61aeae6-218f-4950-a7ec-d69aa07c502a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564745758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26772024-02-15T02:58:17.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564791782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26782024-02-15T02:58:17.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564840996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26792024-02-15T02:58:17.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564868972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26802024-02-15T02:58:17.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564899838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26812024-02-15T02:58:17.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564916506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26822024-02-15T02:58:17.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564939953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26832024-02-15T02:58:17.665Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.564961581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26842024-02-15T02:58:17.665Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565024183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26852024-02-15T02:58:17.665Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565087715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26862024-02-15T02:58:17.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565116981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26872024-02-15T02:58:17.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565159996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26882024-02-15T02:58:17.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565212909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26892024-02-15T02:58:17.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565870027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26902024-02-15T02:58:17.667Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565909962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26912024-02-15T02:58:17.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565936858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26922024-02-15T02:58:17.667Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.565969904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26932024-02-15T02:58:17.667Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.566445275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26942024-02-15T02:58:17.667Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.566625382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26952024-02-15T02:58:17.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567021213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26962024-02-15T02:58:17.667Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567071696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26972024-02-15T02:58:17.667Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567094873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26982024-02-15T02:58:17.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567237315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
26992024-02-15T02:58:17.667Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567292488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27002024-02-15T02:58:17.667Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567329064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27012024-02-15T02:58:17.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567344082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27022024-02-15T02:58:17.667Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567366829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27032024-02-15T02:58:17.668Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.567806334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27042024-02-15T02:58:17.668Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568292443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27052024-02-15T02:58:17.668Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568332238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27062024-02-15T02:58:17.668Z{"msg":"[0]R flush_numbers[0..12]: [25, 40, 40, 42, 39, 43, 40, 40, 36, 35, 44, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568358525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27072024-02-15T02:58:17.668Z{"msg":"[0]R generation[0..12]: [14, 22, 22, 22, 21, 22, 22, 22, 19, 18, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568382642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27082024-02-15T02:58:17.668Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.56839918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27092024-02-15T02:58:17.668Z{"msg":"[1]R flush_numbers[0..12]: [25, 40, 40, 42, 39, 43, 40, 40, 36, 35, 44, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568427726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27102024-02-15T02:58:17.668Z{"msg":"[1]R generation[0..12]: [14, 22, 22, 22, 21, 22, 22, 22, 19, 18, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568442264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27112024-02-15T02:58:17.668Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568463182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27122024-02-15T02:58:17.668Z{"msg":"[2]R flush_numbers[0..12]: [25, 40, 40, 42, 39, 43, 40, 40, 36, 35, 44, 40]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
27132024-02-15T02:58:17.668Z,"time":"2024-02-15T02:58:16.56847749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27142024-02-15T02:58:17.669Z{"msg":"[2]R generation[0..12]: [14, 22, 22, 22, 21, 22, 22, 22, 19, 18, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568491508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27152024-02-15T02:58:17.669Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568507006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27162024-02-15T02:58:17.669Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568528004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27172024-02-15T02:58:17.669Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568542612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27182024-02-15T02:58:17.669Z{"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568568309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27192024-02-15T02:58:17.669Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568582377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27202024-02-15T02:58:17.669Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568602715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27212024-02-15T02:58:17.669Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568615493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27222024-02-15T02:58:17.669Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568629181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27232024-02-15T02:58:17.669Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568650238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27242024-02-15T02:58:17.669Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568675205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27252024-02-15T02:58:17.670Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568691483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27262024-02-15T02:58:17.670Z{"msg":"a61aeae6-218f-4950-a7ec-d69aa07c502a is now active with session: 09d9cab4-fd25-4fd2-a92b-3d78788346e8","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568712421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27272024-02-15T02:58:17.670Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568725199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231,"session_id":"09d9cab4-fd25-4fd2-a92b-3d78788346e8"}
27282024-02-15T02:58:17.670Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:16.568749546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231}
27292024-02-15T02:58:17.670ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
27302024-02-15T02:58:17.670ZLoading write count information from file "/tmp/test_repair_verify.data"
27312024-02-15T02:58:17.670ZRun Repair workload
27322024-02-15T02:58:17.670Z01/30 Write block 405 len 24576 data: 2 2 3 3 4 4
27332024-02-15T02:58:17.670Z02/30 Write block 96 len 8192 data: 5 6
27342024-02-15T02:58:17.670Z03/30 Write block 331 len 24576 data: 4 3 4 4 6 5
27352024-02-15T02:58:17.670Z04/30 Write block 219 len 16384 data: 5 6 5 5
27362024-02-15T02:58:17.671Z05/30 Read block 316 len 16384
27372024-02-15T02:58:17.671Z06/30 Read block 44 len 24576
27382024-02-15T02:58:17.691Z07/30 Write block 527 len 16384 data: 3 3 3 3
27392024-02-15T02:58:17.691Z08/30 Write block 147 len 32768 data: 4 4 3 3 3 3 3 2
27402024-02-15T02:58:17.691Z09/30 Write block 165 len 36864 data: 3 3 3 3 3 3 3 2 2
27412024-02-15T02:58:17.691Z10/30 Flush
27422024-02-15T02:58:17.692Z11/30 Write block 378 len 4096 data: 4
27432024-02-15T02:58:17.692Z12/30 Flush
27442024-02-15T02:58:17.692Z13/30 Write block 574 len 20480 data: 3 4 4 3 3
27452024-02-15T02:58:17.693Z14/30 Read block 548 len 32768
27462024-02-15T02:58:17.693Z15/30 Flush
27472024-02-15T02:58:17.693Z16/30 Write block 332 len 28672 data: 4 5 5 7 6 5 5
27482024-02-15T02:58:17.693Z17/30 Write block 81 len 28672 data: 3 3 2 2 2 2 3
27492024-02-15T02:58:17.694Z18/30 Write block 356 len 16384 data: 2 2 2 2
27502024-02-15T02:58:17.694Z19/30 Read block 470 len 28672
27512024-02-15T02:58:17.694Z20/30 Write block 223 len 24576 data: 4 4 4 6 6 6
27522024-02-15T02:58:17.695Z21/30 Flush
27532024-02-15T02:58:17.695Z22/30 Write block 366 len 36864 data: 3 3 2 2 2 2 2 2 2
27542024-02-15T02:58:17.695Z23/30 Read block 238 len 32768
27552024-02-15T02:58:17.695Z24/30 Read block 391 len 4096
27562024-02-15T02:58:17.695Z25/30 Read block 477 len 16384
27572024-02-15T02:58:17.695Z26/30 Read block 254 len 8192
27582024-02-15T02:58:17.695Z27/30 Read block 418 len 4096
27592024-02-15T02:58:17.696Z28/30 Read block 90 len 12288
27602024-02-15T02:58:17.696Z29/30 Write block 115 len 32768 data: 4 4 4 3 2 2 3 2
27612024-02-15T02:58:17.696Z30/30 Write block 384 len 4096 data: 4
27622024-02-15T02:58:17.696Z----------------------------------------------------------------
27632024-02-15T02:58:17.697Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
27642024-02-15T02:58:17.697ZStates: Active Active Active
27652024-02-15T02:58:17.697ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27662024-02-15T02:58:17.698Z 1 Acked 1000 Write 6 Done Done Done false
27672024-02-15T02:58:17.698Z 2 Acked 1001 Write 2 Done Done Done false
27682024-02-15T02:58:17.698Z 3 Acked 1002 Write 6 Done Done Sent false
27692024-02-15T02:58:17.698Z 4 Acked 1003 Write 4 Done Done Sent false
27702024-02-15T02:58:17.698Z 5 Acked 1004 Read 4 Done Done Sent false
27712024-02-15T02:58:17.698Z 6 Acked 1005 Read 6 Done Done Sent false
27722024-02-15T02:58:17.698Z 7 Acked 1006 Write 4 Done Done Sent false
27732024-02-15T02:58:17.699Z 8 Acked 1007 Write 8 Done Done Sent false
27742024-02-15T02:58:17.699Z 9 Acked 1008 Write 9 Done Done Sent false
27752024-02-15T02:58:17.699Z 10 Acked 1009 Flush 0 Done Done Sent false
27762024-02-15T02:58:17.699Z 11 Acked 1010 Write 1 Done Done Sent false
27772024-02-15T02:58:17.699Z 12 Acked 1011 Flush 0 Done Done Sent false
27782024-02-15T02:58:17.699Z 13 Acked 1012 Write 5 Done Done Sent false
27792024-02-15T02:58:17.700Z 14 Acked 1013 Read 8 Done Done Sent false
27802024-02-15T02:58:17.700Z 15 Acked 1014 Flush 0 Done Done Sent false
27812024-02-15T02:58:17.700Z 16 Acked 1015 Write 7 Done Done Sent false
27822024-02-15T02:58:17.700Z 17 Acked 1016 Write 7 Done Done Sent false
27832024-02-15T02:58:17.700Z 18 Acked 1017 Write 4 Done Done Sent false
27842024-02-15T02:58:17.700Z 19 Acked 1018 Read 7 Done Done Sent false
27852024-02-15T02:58:17.700Z 20 Acked 1019 Write 6 Done Done Sent false
27862024-02-15T02:58:17.700Z 21 Acked 1020 Flush 0 Done Done Sent false
27872024-02-15T02:58:17.701Z 22 Acked 1021 Write 9 Done Done Sent false
27882024-02-15T02:58:17.701Z 23 Acked 1022 Read 8 Done Done Sent false
27892024-02-15T02:58:17.701Z 24 Acked 1023 Read 1 Done Done Sent false
27902024-02-15T02:58:17.701Z 25 Acked 1024 Read 4 Done Done Sent false
27912024-02-15T02:58:17.701Z 26 Acked 1025 Read 2 Done Done Sent false
27922024-02-15T02:58:17.701Z 27 Acked 1026 Read 1 Done Done Sent false
27932024-02-15T02:58:17.702Z 28 Acked 1027 Read 3 Done Done Sent false
27942024-02-15T02:58:17.702Z 29 Acked 1028 Write 8 Sent Sent Sent false
27952024-02-15T02:58:17.702Z 30 Acked 1029 Write 1 Sent Sent Sent false
27962024-02-15T02:58:17.702Z STATES DS:0 DS:1 DS:2 TOTAL
27972024-02-15T02:58:17.702Z New 0 0 0 0
27982024-02-15T02:58:17.702Z Sent 2 2 28 32
27992024-02-15T02:58:17.702Z Done 28 28 2 58
28002024-02-15T02:58:17.702Z Skipped 0 0 0 0
28012024-02-15T02:58:17.702Z Error 0 0 0 0
28022024-02-15T02:58:17.702ZLast Flush: 1020 1020 0
28032024-02-15T02:58:17.702ZDownstairs last five completed:
28042024-02-15T02:58:17.702ZUpstairs last five completed: 30 29 28 27 26
28052024-02-15T02:58:17.703Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:16.60079944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1231}
28062024-02-15T02:58:17.703ZWrote out file "/tmp/test_repair_verify.data"
28072024-02-15T02:58:17.703Z
28082024-02-15T02:58:17.703ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
28092024-02-15T02:58:17.703ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
28102024-02-15T02:58:17.703Z 0 000-019 14 14 14 25 25 25 F F F
28112024-02-15T02:58:17.703Z 1 020-039 22 22 22 40 40 40 F F F
28122024-02-15T02:58:17.703Z 2 040-059 22 22 22 40 40 40 F F F
28132024-02-15T02:58:17.703Z 3 060-079 22 22 22 42 42 42 F F F
28142024-02-15T02:58:17.703Z 4 080-099 24 24 21 48 48 39 F F T <---
28152024-02-15T02:58:17.703Z 5 100-119 22 22 22 43 43 43 T T F <---
28162024-02-15T02:58:17.703Z 6 120-139 22 22 22 40 40 40 T T F <---
28172024-02-15T02:58:17.704Z 7 140-159 24 24 22 45 45 40 F F F <---
28182024-02-15T02:58:17.704Z 8 160-179 24 24 19 45 45 36 F F F <---
28192024-02-15T02:58:17.704Z 9 180-199 18 18 18 35 35 35 F F F
28202024-02-15T02:58:17.704Z 10 200-219 24 24 23 45 45 44 F F F <---
28212024-02-15T02:58:17.704Z 11 220-239 24 24 22 48 48 40 F F F <---
28222024-02-15T02:58:17.704Z 12 240-259 15 15 15 28 28 28 F F F
28232024-02-15T02:58:17.704Z 13 260-279 15 15 15 28 28 28 F F F
28242024-02-15T02:58:17.704Z 14 280-299 22 22 22 41 41 41 F F F
28252024-02-15T02:58:17.704Z 15 300-319 16 16 16 33 33 33 F F F
28262024-02-15T02:58:17.704Z 16 320-339 24 24 18 48 48 35 F F F <---
28272024-02-15T02:58:17.704Z 17 340-359 24 24 21 48 48 39 F F F <---
28282024-02-15T02:58:17.705Z 18 360-379 24 24 18 46 46 35 T T F <---
28292024-02-15T02:58:17.705Z 19 380-399 15 15 15 28 28 28 T T F <---
28302024-02-15T02:58:17.705Z 20 400-419 24 24 21 45 45 39 F F T <---
28312024-02-15T02:58:17.705Z 21 420-439 23 23 23 44 44 44 F F F
28322024-02-15T02:58:17.705Z 22 440-459 21 21 21 39 39 39 F F F
28332024-02-15T02:58:17.705Z 23 460-479 22 22 22 41 41 41 F F F
28342024-02-15T02:58:17.705Z 24 480-499 22 22 22 41 41 41 F F F
28352024-02-15T02:58:17.705Z 25 500-519 22 22 22 40 40 40 F F F
28362024-02-15T02:58:17.706Z 26 520-539 24 24 22 45 45 43 F F F <---
28372024-02-15T02:58:17.706Z 27 540-559 21 21 21 39 39 39 F F F
28382024-02-15T02:58:17.712Z 28 560-579 24 24 21 47 47 39 F F F <---
28392024-02-15T02:58:17.712Z 29 580-599 22 22 22 41 41 41 F F F
28402024-02-15T02:58:17.713ZMax gen: 24, Max flush: 48
28412024-02-15T02:58:17.713ZError: Difference in extent metadata found!
28422024-02-15T02:58:17.713ZA Difference in extent metadata is expected here
28432024-02-15T02:58:17.713ZOn loop 12
28442024-02-15T02:58:17.713Z
28452024-02-15T02:58:17.713ZVerifying data now
28462024-02-15T02:58:23.368ZNo change after verify
28472024-02-15T02:58:23.368ZLoop: 12 Downstairs dump after verify (and repair):
28482024-02-15T02:58:23.384ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
28492024-02-15T02:58:23.384Z 0 000-019 14 14 14 25 25 25 F F F
28502024-02-15T02:58:23.385Z 1 020-039 22 22 22 40 40 40 F F F
28512024-02-15T02:58:23.385Z 2 040-059 22 22 22 40 40 40 F F F
28522024-02-15T02:58:23.385Z 3 060-079 22 22 22 42 42 42 F F F
28532024-02-15T02:58:23.385Z 4 080-099 24 24 24 48 48 48 F F F
28542024-02-15T02:58:23.385Z 5 100-119 25 25 25 49 49 49 F F F
28552024-02-15T02:58:23.385Z 6 120-139 25 25 25 49 49 49 F F F
28562024-02-15T02:58:23.385Z 7 140-159 24 24 24 45 45 45 F F F
28572024-02-15T02:58:23.385Z 8 160-179 24 24 24 45 45 45 F F F
28582024-02-15T02:58:23.385Z 9 180-199 18 18 18 35 35 35 F F F
28592024-02-15T02:58:23.385Z 10 200-219 24 24 24 45 45 45 F F F
28602024-02-15T02:58:23.385Z 11 220-239 24 24 24 48 48 48 F F F
28612024-02-15T02:58:23.385Z 12 240-259 15 15 15 28 28 28 F F F
28622024-02-15T02:58:23.385Z 13 260-279 15 15 15 28 28 28 F F F
28632024-02-15T02:58:23.389Z 14 280-299 22 22 22 41 41 41 F F F
28642024-02-15T02:58:23.389Z 15 300-319 16 16 16 33 33 33 F F F
28652024-02-15T02:58:23.389Z 16 320-339 24 24 24 48 48 48 F F F
28662024-02-15T02:58:23.389Z 17 340-359 24 24 24 48 48 48 F F F
28672024-02-15T02:58:23.389Z 18 360-379 25 25 25 49 49 49 F F F
28682024-02-15T02:58:23.389Z 19 380-399 25 25 25 49 49 49 F F F
28692024-02-15T02:58:23.389Z 20 400-419 24 24 24 45 45 45 F F F
28702024-02-15T02:58:23.389Z 21 420-439 23 23 23 44 44 44 F F F
28712024-02-15T02:58:23.389Z 22 440-459 21 21 21 39 39 39 F F F
28722024-02-15T02:58:23.389Z 23 460-479 22 22 22 41 41 41 F F F
28732024-02-15T02:58:23.389Z 24 480-499 22 22 22 41 41 41 F F F
28742024-02-15T02:58:23.389Z 25 500-519 22 22 22 40 40 40 F F F
28752024-02-15T02:58:23.389Z 26 520-539 24 24 24 45 45 45 F F F
28762024-02-15T02:58:23.389Z 27 540-559 21 21 21 39 39 39 F F F
28772024-02-15T02:58:23.389Z 28 560-579 24 24 24 47 47 47 F F F
28782024-02-15T02:58:23.389Z 29 580-599 22 22 22 41 41 41 F F F
28792024-02-15T02:58:23.389ZMax gen: 25, Max flush: 49
28802024-02-15T02:58:23.389Z
28812024-02-15T02:58:23.389ZBegin loop 13
28822024-02-15T02:58:23.389ZDownstairs to restart: 0
28832024-02-15T02:58:23.408Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.356976598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239}
28842024-02-15T02:58:23.408Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357276641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239}
28852024-02-15T02:58:23.408Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
28862024-02-15T02:58:23.409Z,"time":"2024-02-15T02:58:22.357326015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239}
28872024-02-15T02:58:23.425Z{"msg":"Crucible 06bf9e73-d163-4045-bd04-cfe7b58eb2d4 has session id: 3aeda1b2-397d-4e04-83da-83edb2050458","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357367469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28882024-02-15T02:58:23.425Z{"msg":"Upstairs opts: Upstairs UUID: 06bf9e73-d163-4045-bd04-cfe7b58eb2d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357425552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28892024-02-15T02:58:23.425Z{"msg":"Crucible stats registered with UUID: 06bf9e73-d163-4045-bd04-cfe7b58eb2d4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357467887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28902024-02-15T02:58:23.425Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357497793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239}
28912024-02-15T02:58:23.425Z{"msg":"06bf9e73-d163-4045-bd04-cfe7b58eb2d4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357543927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28922024-02-15T02:58:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357576883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28932024-02-15T02:58:23.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357596301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28942024-02-15T02:58:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357641655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28952024-02-15T02:58:23.426Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357676971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28962024-02-15T02:58:23.426Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357715596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28972024-02-15T02:58:23.426Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357742582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28982024-02-15T02:58:23.426Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357788617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
28992024-02-15T02:58:23.426Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357827422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29002024-02-15T02:58:23.426Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357876336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29012024-02-15T02:58:23.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.357932249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29022024-02-15T02:58:23.426Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:22.358026877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29032024-02-15T02:58:23.426Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:22.358072591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29042024-02-15T02:58:23.426Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:22.358128234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29052024-02-15T02:58:23.426Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358201385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29062024-02-15T02:58:23.426Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358249559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29072024-02-15T02:58:23.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358284765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29082024-02-15T02:58:23.426Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358309631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29092024-02-15T02:58:23.426Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358825807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29102024-02-15T02:58:23.426Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358865662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29112024-02-15T02:58:23.427Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358888599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29122024-02-15T02:58:23.427Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.358912036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29132024-02-15T02:58:23.427Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.359392106Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29142024-02-15T02:58:23.427Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.359499593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29152024-02-15T02:58:23.427Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.359962834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29162024-02-15T02:58:23.427Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.360006369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29172024-02-15T02:58:23.427Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.360029996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29182024-02-15T02:58:23.427Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.360054103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29192024-02-15T02:58:23.427Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.36007629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29202024-02-15T02:58:23.427Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:22.360114855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29212024-02-15T02:58:24.419Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.359824225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29222024-02-15T02:58:24.429Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.359982325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29232024-02-15T02:58:24.429Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.360810941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29242024-02-15T02:58:24.429Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.360861545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29252024-02-15T02:58:24.429Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.361334256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29262024-02-15T02:58:24.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.361884077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29272024-02-15T02:58:24.429Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.361926181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29282024-02-15T02:58:24.430Z{"msg":"[0]R flush_numbers[0..12]: [25, 40, 40, 42, 48, 49, 49, 45, 45, 35, 45, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.361952398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29292024-02-15T02:58:24.430Z{"msg":"[0]R generation[0..12]: [14, 22, 22, 22, 24, 25, 25, 24, 24, 18, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.361976275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29302024-02-15T02:58:24.430Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362006771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29312024-02-15T02:58:24.430Z{"msg":"[1]R flush_numbers[0..12]: [25, 40, 40, 42, 48, 49, 49, 45, 45, 35, 45, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362032148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29322024-02-15T02:58:24.430Z{"msg":"[1]R generation[0..12]: [14, 22, 22, 22, 24, 25, 25, 24, 24, 18, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362046996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29332024-02-15T02:58:24.430Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362068724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29342024-02-15T02:58:24.430Z{"msg":"[2]R flush_numbers[0..12]: [25, 40, 40, 42, 48, 49, 49, 45, 45, 35, 45, 48]","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO
29352024-02-15T02:58:24.430Z"2024-02-15T02:58:23.362084392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29362024-02-15T02:58:24.430Z{"msg":"[2]R generation[0..12]: [14, 22, 22, 22, 24, 25, 25, 24, 24, 18, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362168081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29372024-02-15T02:58:24.430Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362203097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29382024-02-15T02:58:24.430Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362227904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29392024-02-15T02:58:24.430Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362242502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29402024-02-15T02:58:24.431Z{"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362262849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29412024-02-15T02:58:24.431Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362277377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29422024-02-15T02:58:24.431Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362290926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29432024-02-15T02:58:24.431Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362310793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29442024-02-15T02:58:24.431Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362324622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29452024-02-15T02:58:24.431Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.3623383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29462024-02-15T02:58:24.431Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362371536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29472024-02-15T02:58:24.431Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362407571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29482024-02-15T02:58:24.431Z{"msg":"06bf9e73-d163-4045-bd04-cfe7b58eb2d4 is now active with session: 3aeda1b2-397d-4e04-83da-83edb2050458","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362443537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29492024-02-15T02:58:24.431Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362479792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239,"session_id":"3aeda1b2-397d-4e04-83da-83edb2050458"}
29502024-02-15T02:58:24.432Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:23.362514128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239}
29512024-02-15T02:58:24.432ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
29522024-02-15T02:58:24.432ZLoading write count information from file "/tmp/test_repair_verify.data"
29532024-02-15T02:58:24.432ZRun Repair workload
29542024-02-15T02:58:24.432Z01/30 Write block 589 len 12288 data: 5 4 4
29552024-02-15T02:58:24.432Z02/30 Flush
29562024-02-15T02:58:24.432Z03/30 Write block 235 len 12288 data: 3 3 2
29572024-02-15T02:58:24.432Z04/30 Read block 66 len 20480
29582024-02-15T02:58:24.432Z05/30 Write block 548 len 20480 data: 2 2 2 2 2
29592024-02-15T02:58:24.432Z06/30 Read block 39 len 36864
29602024-02-15T02:58:24.433Z07/30 Flush
29612024-02-15T02:58:24.442Z08/30 Write block 499 len 28672 data: 2 3 4 4 4 4 4
29622024-02-15T02:58:24.442Z09/30 Read block 142 len 36864
29632024-02-15T02:58:24.442Z10/30 Write block 275 len 4096 data: 3
29642024-02-15T02:58:24.442Z11/30 Read block 38 len 12288
29652024-02-15T02:58:24.442Z12/30 Write block 344 len 36864 data: 2 2 2 3 3 3 3 3 3
29662024-02-15T02:58:24.442Z13/30 Read block 552 len 8192
29672024-02-15T02:58:24.442Z14/30 Read block 529 len 32768
29682024-02-15T02:58:24.443Z15/30 Read block 62 len 20480
29692024-02-15T02:58:24.443Z16/30 Read block 294 len 12288
29702024-02-15T02:58:24.443Z17/30 Write block 317 len 40960 data: 3 4 3 3 2 2 3 3 3 3
29712024-02-15T02:58:24.443Z18/30 Read block 66 len 24576
29722024-02-15T02:58:24.443Z19/30 Write block 193 len 24576 data: 5 5 5 4 3 3
29732024-02-15T02:58:24.443Z20/30 Read block 298 len 12288
29742024-02-15T02:58:24.443Z21/30 Read block 538 len 4096
29752024-02-15T02:58:24.443Z22/30 Read block 425 len 20480
29762024-02-15T02:58:24.443Z23/30 Flush
29772024-02-15T02:58:24.443Z24/30 Flush
29782024-02-15T02:58:24.452Z25/30 Write block 435 len 36864 data: 2 2 2 2 3 3 3 4 5
29792024-02-15T02:58:24.452Z26/30 Write block 352 len 32768 data: 4 3 2 2 3 3 3 3
29802024-02-15T02:58:24.452Z27/30 Write block 20 len 12288 data: 3 2 2
29812024-02-15T02:58:24.452Z28/30 Read block 191 len 32768
29822024-02-15T02:58:24.452Z29/30 Read block 471 len 12288
29832024-02-15T02:58:24.452Z30/30 Write block 296 len 40960 data: 2 2 2 3 4 4 4 5 6 7
29842024-02-15T02:58:24.452Z----------------------------------------------------------------
29852024-02-15T02:58:24.452Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:28
29862024-02-15T02:58:24.453ZStates: Active Active Active
29872024-02-15T02:58:24.453ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29882024-02-15T02:58:24.453Z 3 Acked 1002 Write 3 Done Done Done false
29892024-02-15T02:58:24.453Z 4 Acked 1003 Read 5 Sent Done Done false
29902024-02-15T02:58:24.453Z 5 Acked 1004 Write 5 Sent Done Done false
29912024-02-15T02:58:24.453Z 6 Acked 1005 Read 9 Sent Done Done false
29922024-02-15T02:58:24.453Z 7 Acked 1006 Flush 0 Sent Done Done false
29932024-02-15T02:58:24.453Z 8 Acked 1007 Write 7 Sent Done Done false
29942024-02-15T02:58:24.453Z 9 Acked 1008 Read 9 Sent Done Done false
29952024-02-15T02:58:24.454Z 10 Acked 1009 Write 1 Sent Done Done false
29962024-02-15T02:58:24.454Z 11 Acked 1010 Read 3 Sent Done Done false
29972024-02-15T02:58:24.454Z 12 Acked 1011 Write 9 Sent Done Done false
29982024-02-15T02:58:24.454Z 13 Acked 1012 Read 2 Sent Done Done false
29992024-02-15T02:58:24.454Z 14 Acked 1013 Read 8 Sent Done Done false
30002024-02-15T02:58:24.454Z 15 Acked 1014 Read 5 Sent Done Done false
30012024-02-15T02:58:24.454Z 16 Acked 1015 Read 3 Sent Done Done false
30022024-02-15T02:58:24.454Z 17 Acked 1016 Write 10 Sent Done Done false
30032024-02-15T02:58:24.454Z 18 Acked 1017 Read 6 Sent Done Done false
30042024-02-15T02:58:24.454Z 19 Acked 1018 Write 6 Sent Done Done false
30052024-02-15T02:58:24.454Z 20 Acked 1019 Read 3 Sent Done Done false
30062024-02-15T02:58:24.454Z 21 Acked 1020 Read 1 Sent Done Done false
30072024-02-15T02:58:24.454Z 22 Acked 1021 Read 5 Sent Done Done false
30082024-02-15T02:58:24.455Z 23 Acked 1022 Flush 0 Sent Done Done false
30092024-02-15T02:58:24.455Z 24 Acked 1023 Flush 0 Sent Done Done false
30102024-02-15T02:58:24.455Z 25 Acked 1024 Write 9 Sent Done Done false
30112024-02-15T02:58:24.455Z 26 Acked 1025 Write 8 Sent Done Done false
30122024-02-15T02:58:24.455Z 27 Acked 1026 Write 3 Sent Done Done false
30132024-02-15T02:58:24.455Z 28 Acked 1027 Read 8 Sent Done Done false
30142024-02-15T02:58:24.455Z 29 Acked 1028 Read 3 Sent Done Done false
30152024-02-15T02:58:24.455Z 30 Acked 1029 Write 10 Sent Sent Sent false
30162024-02-15T02:58:24.455Z STATES DS:0 DS:1 DS:2 TOTAL
30172024-02-15T02:58:24.455Z New 0 0 0 0
30182024-02-15T02:58:24.455Z Sent 27 1 1 29
30192024-02-15T02:58:24.455Z Done 1 27 27 55
30202024-02-15T02:58:24.456Z Skipped 0 0 0 0
30212024-02-15T02:58:24.456Z Error 0 0 0 0
30222024-02-15T02:58:24.456ZLast Flush: 1001 1023 1023
30232024-02-15T02:58:24.456ZDownstairs last five completed: 1001 1000
30242024-02-15T02:58:24.456ZUpstairs last five completed: 30 29 28 27 26
30252024-02-15T02:58:24.456Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:23.394951643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1239}
30262024-02-15T02:58:24.457ZWrote out file "/tmp/test_repair_verify.data"
30272024-02-15T02:58:24.462Z
30282024-02-15T02:58:24.462ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
30292024-02-15T02:58:24.477ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30302024-02-15T02:58:24.478Z 0 000-019 14 14 14 25 25 25 F F F
30312024-02-15T02:58:24.478Z 1 020-039 22 22 22 40 40 40 F T T <---
30322024-02-15T02:58:24.478Z 2 040-059 22 22 22 40 40 40 F F F
30332024-02-15T02:58:24.478Z 3 060-079 22 22 22 42 42 42 F F F
30342024-02-15T02:58:24.478Z 4 080-099 24 24 24 48 48 48 F F F
30352024-02-15T02:58:24.478Z 5 100-119 25 25 25 49 49 49 F F F
30362024-02-15T02:58:24.478Z 6 120-139 25 25 25 49 49 49 F F F
30372024-02-15T02:58:24.478Z 7 140-159 24 24 24 45 45 45 F F F
30382024-02-15T02:58:24.483Z 8 160-179 24 24 24 45 45 45 F F F
30392024-02-15T02:58:24.483Z 9 180-199 18 26 26 35 52 52 F F F <---
30402024-02-15T02:58:24.483Z 10 200-219 24 24 24 45 45 45 F F F
30412024-02-15T02:58:24.483Z 11 220-239 24 26 26 48 51 51 T F F <---
30422024-02-15T02:58:24.484Z 12 240-259 15 15 15 28 28 28 F F F
30432024-02-15T02:58:24.484Z 13 260-279 15 26 26 28 52 52 F F F <---
30442024-02-15T02:58:24.484Z 14 280-299 22 22 22 41 41 41 F T T <---
30452024-02-15T02:58:24.484Z 15 300-319 16 26 26 33 52 52 F T T <---
30462024-02-15T02:58:24.485Z 16 320-339 24 26 26 48 52 52 F F F <---
30472024-02-15T02:58:24.485Z 17 340-359 24 26 26 48 52 52 F T T <---
30482024-02-15T02:58:24.485Z 18 360-379 25 25 25 49 49 49 F F F
30492024-02-15T02:58:24.485Z 19 380-399 25 25 25 49 49 49 F F F
30502024-02-15T02:58:24.485Z 20 400-419 24 24 24 45 45 45 F F F
30512024-02-15T02:58:24.486Z 21 420-439 23 23 23 44 44 44 F T T <---
30522024-02-15T02:58:24.486Z 22 440-459 21 21 21 39 39 39 F T T <---
30532024-02-15T02:58:24.486Z 23 460-479 22 22 22 41 41 41 F F F
30542024-02-15T02:58:24.486Z 24 480-499 22 26 26 41 52 52 F F F <---
30552024-02-15T02:58:24.486Z 25 500-519 22 26 26 40 52 52 F F F <---
30562024-02-15T02:58:24.486Z 26 520-539 24 24 24 45 45 45 F F F
30572024-02-15T02:58:24.487Z 27 540-559 21 26 26 39 51 51 F F F <---
30582024-02-15T02:58:24.487Z 28 560-579 24 24 24 47 47 47 F F F
30592024-02-15T02:58:24.487Z 29 580-599 26 26 26 50 50 50 F F F
30602024-02-15T02:58:24.487ZMax gen: 26, Max flush: 52
30612024-02-15T02:58:24.488ZError: Difference in extent metadata found!
30622024-02-15T02:58:24.488ZA Difference in extent metadata is expected here
30632024-02-15T02:58:24.488ZOn loop 13
30642024-02-15T02:58:24.488Z
30652024-02-15T02:58:24.489ZVerifying data now
30662024-02-15T02:58:29.125ZNo change after verify
30672024-02-15T02:58:29.143ZLoop: 13 Downstairs dump after verify (and repair):
30682024-02-15T02:58:29.143ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30692024-02-15T02:58:29.143Z 0 000-019 14 14 14 25 25 25 F F F
30702024-02-15T02:58:29.143Z 1 020-039 27 27 27 53 53 53 F F F
30712024-02-15T02:58:29.144Z 2 040-059 22 22 22 40 40 40 F F F
30722024-02-15T02:58:29.144Z 3 060-079 22 22 22 42 42 42 F F F
30732024-02-15T02:58:29.144Z 4 080-099 24 24 24 48 48 48 F F F
30742024-02-15T02:58:29.144Z 5 100-119 25 25 25 49 49 49 F F F
30752024-02-15T02:58:29.144Z 6 120-139 25 25 25 49 49 49 F F F
30762024-02-15T02:58:29.144Z 7 140-159 24 24 24 45 45 45 F F F
30772024-02-15T02:58:29.145Z 8 160-179 24 24 24 45 45 45 F F F
30782024-02-15T02:58:29.145Z 9 180-199 26 26 26 52 52 52 F F F
30792024-02-15T02:58:29.145Z 10 200-219 24 24 24 45 45 45 F F F
30802024-02-15T02:58:29.145Z 11 220-239 26 26 26 51 51 51 F F F
30812024-02-15T02:58:29.145Z 12 240-259 15 15 15 28 28 28 F F F
30822024-02-15T02:58:29.145Z 13 260-279 26 26 26 52 52 52 F F F
30832024-02-15T02:58:29.145Z 14 280-299 27 27 27 53 53 53 F F F
30842024-02-15T02:58:29.145Z 15 300-319 27 27 27 53 53 53 F F F
30852024-02-15T02:58:29.145Z 16 320-339 26 26 26 52 52 52 F F F
30862024-02-15T02:58:29.145Z 17 340-359 27 27 27 53 53 53 F F F
30872024-02-15T02:58:29.145Z 18 360-379 25 25 25 49 49 49 F F F
30882024-02-15T02:58:29.146Z 19 380-399 25 25 25 49 49 49 F F F
30892024-02-15T02:58:29.146Z 20 400-419 24 24 24 45 45 45 F F F
30902024-02-15T02:58:29.146Z 21 420-439 27 27 27 53 53 53 F F F
30912024-02-15T02:58:29.146Z 22 440-459 27 27 27 53 53 53 F F F
30922024-02-15T02:58:29.146Z 23 460-479 22 22 22 41 41 41 F F F
30932024-02-15T02:58:29.147Z 24 480-499 26 26 26 52 52 52 F F F
30942024-02-15T02:58:29.147Z 25 500-519 26 26 26 52 52 52 F F F
30952024-02-15T02:58:29.147Z 26 520-539 24 24 24 45 45 45 F F F
30962024-02-15T02:58:29.147Z 27 540-559 26 26 26 51 51 51 F F F
30972024-02-15T02:58:29.147Z 28 560-579 24 24 24 47 47 47 F F F
30982024-02-15T02:58:29.147Z 29 580-599 26 26 26 50 50 50 F F F
30992024-02-15T02:58:29.147ZMax gen: 27, Max flush: 53
31002024-02-15T02:58:29.158Z
31012024-02-15T02:58:29.158ZBegin loop 14
31022024-02-15T02:58:29.159ZDownstairs to restart: 0
31032024-02-15T02:58:29.190Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106061235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247}
31042024-02-15T02:58:29.190Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106372246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internalCrucible runtime is spawned
31052024-02-15T02:58:29.190Z","pid":1247}
31062024-02-15T02:58:29.190Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106448006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247}
31072024-02-15T02:58:29.190Z{"msg":"Crucible e3aeb8c1-6ffe-4377-b95a-ee68f859f26d has session id: 380c0631-6c03-4d09-8d36-301b5b01bf0a","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106472653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31082024-02-15T02:58:29.190Z{"msg":"Upstairs opts: Upstairs UUID: e3aeb8c1-6ffe-4377-b95a-ee68f859f26d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106513578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31092024-02-15T02:58:29.190Z{"msg":"Crucible stats registered with UUID: e3aeb8c1-6ffe-4377-b95a-ee68f859f26d","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106546694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31102024-02-15T02:58:29.191Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106574621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247}
31112024-02-15T02:58:29.191Z{"msg":"e3aeb8c1-6ffe-4377-b95a-ee68f859f26d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106636663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31122024-02-15T02:58:29.191Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.10666229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31132024-02-15T02:58:29.191Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106689796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31142024-02-15T02:58:29.191Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.10674025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31152024-02-15T02:58:29.191Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106765307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31162024-02-15T02:58:29.191Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106796033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31172024-02-15T02:58:29.191Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106811331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31182024-02-15T02:58:29.191Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106831238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31192024-02-15T02:58:29.191Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106879162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31202024-02-15T02:58:29.192Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.106918058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31212024-02-15T02:58:29.192Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:28.106950534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31222024-02-15T02:58:29.192Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:28.10698231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31232024-02-15T02:58:29.192Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107031443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31242024-02-15T02:58:29.192Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107068919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31252024-02-15T02:58:29.192Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:28.107105584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31262024-02-15T02:58:29.192Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107128331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31272024-02-15T02:58:29.192Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107150928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31282024-02-15T02:58:29.193Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107276293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31292024-02-15T02:58:29.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107763521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31302024-02-15T02:58:29.193Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107803507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31312024-02-15T02:58:29.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107847731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31322024-02-15T02:58:29.194Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.107875907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31332024-02-15T02:58:29.194Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.108450806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31342024-02-15T02:58:29.194Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.10849173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31352024-02-15T02:58:29.194Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.109073828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31362024-02-15T02:58:29.194Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.109112832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31372024-02-15T02:58:29.194Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.109138779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31382024-02-15T02:58:29.195Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.109166456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31392024-02-15T02:58:29.195Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.109191833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31402024-02-15T02:58:29.195Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:28.109224549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31412024-02-15T02:58:30.160Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.10859245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31422024-02-15T02:58:30.166Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.108764229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31432024-02-15T02:58:30.166Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.109573697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31442024-02-15T02:58:30.166Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.109618582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31452024-02-15T02:58:30.166Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110140237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31462024-02-15T02:58:30.167Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110729633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31472024-02-15T02:58:30.167Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110768368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31482024-02-15T02:58:30.167Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 40, 42, 48, 49, 49, 45, 45, 52, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110795984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31492024-02-15T02:58:30.167Z{"msg":"[0]R generation[0..12]: [14, 27, 22, 22, 24, 25, 25, 24, 24, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110820771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31502024-02-15T02:58:30.167Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110836079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31512024-02-15T02:58:30.167Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 40, 42, 48, 49, 49, 45, 45, 52, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110857836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31522024-02-15T02:58:30.167Z{"msg":"[1]R generation[0..12]: [14, 27, 22, 22, 24, 25, 25, 24, 24, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110871945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31532024-02-15T02:58:30.167Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110892532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31542024-02-15T02:58:30.167Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 40, 42, 48, 49, 49, 45, 45, 52, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.11090718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31552024-02-15T02:58:30.167Z{"msg":"[2]R generation[0..12]: [14, 27, 22, 22, 24, 25, 25, 24, 24, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110937597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
31562024-02-15T02:58:30.167Zdownstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31572024-02-15T02:58:30.167Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110959124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31582024-02-15T02:58:30.167Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.110972812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31592024-02-15T02:58:30.167Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.11098699Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31602024-02-15T02:58:30.167Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111006418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31612024-02-15T02:58:30.167Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111027515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31622024-02-15T02:58:30.167Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111041964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31632024-02-15T02:58:30.167Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111054872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31642024-02-15T02:58:30.167Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111076679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31652024-02-15T02:58:30.167Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111090407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31662024-02-15T02:58:30.167Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111104426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31672024-02-15T02:58:30.167Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111117724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31682024-02-15T02:58:30.168Z{"msg":"e3aeb8c1-6ffe-4377-b95a-ee68f859f26d is now active with session: 380c0631-6c03-4d09-8d36-301b5b01bf0a","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111130762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31692024-02-15T02:58:30.168Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.1111497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247,"session_id":"380c0631-6c03-4d09-8d36-301b5b01bf0a"}
31702024-02-15T02:58:30.168Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:29.111162268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247}
31712024-02-15T02:58:30.168ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
31722024-02-15T02:58:30.168ZLoading write count information from file "/tmp/test_repair_verify.data"
31732024-02-15T02:58:30.168ZRun Repair workload
31742024-02-15T02:58:30.172Z01/30 Write block 257 len 36864 data: 3 3 4 4 4 3 3 4 5
31752024-02-15T02:58:30.172Z02/30 Read block 461 len 12288
31762024-02-15T02:58:30.172Z03/30 Read block 303 len 40960
31772024-02-15T02:58:30.172Z04/30 Write block 257 len 36864 data: 4 4 5 5 5 4 4 5 6
31782024-02-15T02:58:30.172Z05/30 Read block 114 len 4096
31792024-02-15T02:58:30.172Z06/30 Write block 95 len 36864 data: 4 6 7 6 5 4 5 5 4
31802024-02-15T02:58:30.172Z07/30 Read block 292 len 4096
31812024-02-15T02:58:30.176Z08/30 Write block 92 len 28672 data: 3 3 5 5 7 8 7
31822024-02-15T02:58:30.176Z09/30 Read block 597 len 8192
31832024-02-15T02:58:30.176Z10/30 Read block 296 len 8192
31842024-02-15T02:58:30.176Z11/30 Read block 255 len 8192
31852024-02-15T02:58:30.176Z12/30 Flush
31862024-02-15T02:58:30.179Z13/30 Write block 353 len 40960 data: 4 3 3 4 4 4 4 3 3 3
31872024-02-15T02:58:30.179Z14/30 Read block 140 len 28672
31882024-02-15T02:58:30.181Z15/30 Flush
31892024-02-15T02:58:30.181Z16/30 Write block 422 len 16384 data: 2 2 3 3
31902024-02-15T02:58:30.184Z17/30 Write block 470 len 8192 data: 4 4
31912024-02-15T02:58:30.184Z18/30 Read block 111 len 28672
31922024-02-15T02:58:30.184Z19/30 Read block 28 len 32768
31932024-02-15T02:58:30.187Z20/30 Read block 472 len 24576
31942024-02-15T02:58:30.187Z21/30 Write block 181 len 24576 data: 2 2 3 4 4 4
31952024-02-15T02:58:30.188Z22/30 Write block 527 len 16384 data: 4 4 4 4
31962024-02-15T02:58:30.188Z23/30 Read block 186 len 28672
31972024-02-15T02:58:30.191Z24/30 Read block 170 len 12288
31982024-02-15T02:58:30.191Z25/30 Read block 149 len 16384
31992024-02-15T02:58:30.191Z26/30 Read block 329 len 32768
32002024-02-15T02:58:30.191Z27/30 Read block 28 len 4096
32012024-02-15T02:58:30.194Z28/30 Read block 228 len 32768
32022024-02-15T02:58:30.194Z29/30 Write block 399 len 4096 data: 2
32032024-02-15T02:58:30.194Z30/30 Write block 487 len 16384 data: 4 3 4 5
32042024-02-15T02:58:30.194Z----------------------------------------------------------------
32052024-02-15T02:58:30.194Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
32062024-02-15T02:58:30.194ZStates: Active Active Active
32072024-02-15T02:58:30.194ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
32082024-02-15T02:58:30.194Z 1 Acked 1000 Write 9 Done Done Done false
32092024-02-15T02:58:30.195Z 2 Acked 1001 Read 3 Done Done Done false
32102024-02-15T02:58:30.195Z 3 Acked 1002 Read 10 Done Done Done false
32112024-02-15T02:58:30.195Z 4 Acked 1003 Write 9 Done Done Done false
32122024-02-15T02:58:30.195Z 5 Acked 1004 Read 1 Sent Done Done false
32132024-02-15T02:58:30.195Z 6 Acked 1005 Write 9 Sent Done Done false
32142024-02-15T02:58:30.195Z 7 Acked 1006 Read 1 Sent Done Done false
32152024-02-15T02:58:30.195Z 8 Acked 1007 Write 7 Sent Done Done false
32162024-02-15T02:58:30.195Z 9 Acked 1008 Read 2 Sent Done Done false
32172024-02-15T02:58:30.195Z 10 Acked 1009 Read 2 Sent Done Done false
32182024-02-15T02:58:30.195Z 11 Acked 1010 Read 2 Sent Done Done false
32192024-02-15T02:58:30.195Z 12 Acked 1011 Flush 0 Sent Done Done false
32202024-02-15T02:58:30.195Z 13 Acked 1012 Write 10 Sent Done Done false
32212024-02-15T02:58:30.195Z 14 Acked 1013 Read 7 Sent Done Done false
32222024-02-15T02:58:30.195Z 15 Acked 1014 Flush 0 Sent Done Done false
32232024-02-15T02:58:30.195Z 16 Acked 1015 Write 4 Sent Done Done false
32242024-02-15T02:58:30.195Z 17 Acked 1016 Write 2 Sent Done Done false
32252024-02-15T02:58:30.195Z 18 Acked 1017 Read 7 Sent Done Done false
32262024-02-15T02:58:30.195Z 19 Acked 1018 Read 8 Sent Done Done false
32272024-02-15T02:58:30.195Z 20 Acked 1019 Read 6 Sent Done Done false
32282024-02-15T02:58:30.195Z 21 Acked 1020 Write 6 Sent Done Done false
32292024-02-15T02:58:30.195Z 22 Acked 1021 Write 4 Sent Done Done false
32302024-02-15T02:58:30.195Z 23 Acked 1022 Read 7 Sent Done Done false
32312024-02-15T02:58:30.195Z 24 Acked 1023 Read 3 Sent Done Done false
32322024-02-15T02:58:30.196Z 25 Acked 1024 Read 4 Sent Done Done false
32332024-02-15T02:58:30.196Z 26 Acked 1025 Read 8 Sent Done Done false
32342024-02-15T02:58:30.196Z 27 Acked 1026 Read 1 Sent Done Done false
32352024-02-15T02:58:30.196Z 28 Acked 1027 Read 8 Sent Done Done false
32362024-02-15T02:58:30.196Z 29 Acked 1028 Write 1 Sent Done Done false
32372024-02-15T02:58:30.196Z 30 Acked 1029 Write 4 Sent Sent Sent false
32382024-02-15T02:58:30.196Z STATES DS:0 DS:1 DS:2 TOTAL
32392024-02-15T02:58:30.196Z New 0 0 0 0
32402024-02-15T02:58:30.196Z Sent 26 1 1 28
32412024-02-15T02:58:30.196Z Done 4 29 29 62
32422024-02-15T02:58:30.196Z Skipped 0 0 0 0
32432024-02-15T02:58:30.196Z Error 0 0 0 0
32442024-02-15T02:58:30.196ZLast Flush: 0 1014 1014
32452024-02-15T02:58:30.196ZDownstairs last five completed:
32462024-02-15T02:58:30.196ZUpstairs last five completed: 30 29 28 27 26
32472024-02-15T02:58:30.196Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:29.141758715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1247}
32482024-02-15T02:58:30.196ZWrote out file "/tmp/test_repair_verify.data"
32492024-02-15T02:58:30.206Z
32502024-02-15T02:58:30.207ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
32512024-02-15T02:58:30.233ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32522024-02-15T02:58:30.233Z 0 000-019 14 14 14 25 25 25 F F F
32532024-02-15T02:58:30.233Z 1 020-039 27 27 27 53 53 53 F F F
32542024-02-15T02:58:30.233Z 2 040-059 22 22 22 40 40 40 F F F
32552024-02-15T02:58:30.233Z 3 060-079 22 22 22 42 42 42 F F F
32562024-02-15T02:58:30.233Z 4 080-099 24 28 28 48 54 54 F F F <---
32572024-02-15T02:58:30.234Z 5 100-119 25 28 28 49 54 54 F F F <---
32582024-02-15T02:58:30.234Z 6 120-139 25 25 25 49 49 49 F F F
32592024-02-15T02:58:30.234Z 7 140-159 24 24 24 45 45 45 F F F
32602024-02-15T02:58:30.234Z 8 160-179 24 24 24 45 45 45 F F F
32612024-02-15T02:58:30.234Z 9 180-199 26 26 26 52 52 52 F T T <---
32622024-02-15T02:58:30.234Z 10 200-219 24 24 24 45 45 45 F F F
32632024-02-15T02:58:30.235Z 11 220-239 26 26 26 51 51 51 F F F
32642024-02-15T02:58:30.235Z 12 240-259 15 28 28 28 54 54 T F F <---
32652024-02-15T02:58:30.235Z 13 260-279 26 28 28 52 54 54 T F F <---
32662024-02-15T02:58:30.235Z 14 280-299 27 27 27 53 53 53 F F F
32672024-02-15T02:58:30.248Z 15 300-319 27 27 27 53 53 53 F F F
32682024-02-15T02:58:30.248Z 16 320-339 26 26 26 52 52 52 F F F
32692024-02-15T02:58:30.248Z 17 340-359 27 28 28 53 55 55 F F F <---
32702024-02-15T02:58:30.248Z 18 360-379 25 28 28 49 55 55 F F F <---
32712024-02-15T02:58:30.248Z 19 380-399 25 25 25 49 49 49 F T T <---
32722024-02-15T02:58:30.248Z 20 400-419 24 24 24 45 45 45 F F F
32732024-02-15T02:58:30.249Z 21 420-439 27 27 27 53 53 53 F T T <---
32742024-02-15T02:58:30.249Z 22 440-459 27 27 27 53 53 53 F F F
32752024-02-15T02:58:30.249Z 23 460-479 22 22 22 41 41 41 F T T <---
32762024-02-15T02:58:30.249Z 24 480-499 26 26 26 52 52 52 F T T <---
32772024-02-15T02:58:30.249Z 25 500-519 26 26 26 52 52 52 F F F
32782024-02-15T02:58:30.249Z 26 520-539 24 24 24 45 45 45 F T T <---
32792024-02-15T02:58:30.249Z 27 540-559 26 26 26 51 51 51 F F F
32802024-02-15T02:58:30.249Z 28 560-579 24 24 24 47 47 47 F F F
32812024-02-15T02:58:30.249Z 29 580-599 26 26 26 50 50 50 F F F
32822024-02-15T02:58:30.249ZMax gen: 28, Max flush: 55
32832024-02-15T02:58:30.250ZError: Difference in extent metadata found!
32842024-02-15T02:58:30.250ZA Difference in extent metadata is expected here
32852024-02-15T02:58:30.250ZOn loop 14
32862024-02-15T02:58:30.250Z
32872024-02-15T02:58:30.250ZVerifying data now
32882024-02-15T02:58:35.837ZNo change after verify
32892024-02-15T02:58:35.837ZLoop: 14 Downstairs dump after verify (and repair):
32902024-02-15T02:58:35.866ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32912024-02-15T02:58:35.866Z 0 000-019 14 14 14 25 25 25 F F F
32922024-02-15T02:58:35.867Z 1 020-039 27 27 27 53 53 53 F F F
32932024-02-15T02:58:35.867Z 2 040-059 22 22 22 40 40 40 F F F
32942024-02-15T02:58:35.867Z 3 060-079 22 22 22 42 42 42 F F F
32952024-02-15T02:58:35.867Z 4 080-099 28 28 28 54 54 54 F F F
32962024-02-15T02:58:35.867Z 5 100-119 28 28 28 54 54 54 F F F
32972024-02-15T02:58:35.867Z 6 120-139 25 25 25 49 49 49 F F F
32982024-02-15T02:58:35.867Z 7 140-159 24 24 24 45 45 45 F F F
32992024-02-15T02:58:35.868Z 8 160-179 24 24 24 45 45 45 F F F
33002024-02-15T02:58:35.868Z 9 180-199 29 29 29 56 56 56 F F F
33012024-02-15T02:58:35.868Z 10 200-219 24 24 24 45 45 45 F F F
33022024-02-15T02:58:35.868Z 11 220-239 26 26 26 51 51 51 F F F
33032024-02-15T02:58:35.868Z 12 240-259 28 28 28 54 54 54 F F F
33042024-02-15T02:58:35.868Z 13 260-279 28 28 28 54 54 54 F F F
33052024-02-15T02:58:35.869Z 14 280-299 27 27 27 53 53 53 F F F
33062024-02-15T02:58:35.885Z 15 300-319 27 27 27 53 53 53 F F F
33072024-02-15T02:58:35.886Z 16 320-339 26 26 26 52 52 52 F F F
33082024-02-15T02:58:35.886Z 17 340-359 28 28 28 55 55 55 F F F
33092024-02-15T02:58:35.886Z 18 360-379 28 28 28 55 55 55 F F F
33102024-02-15T02:58:35.886Z 19 380-399 29 29 29 56 56 56 F F F
33112024-02-15T02:58:35.886Z 20 400-419 24 24 24 45 45 45 F F F
33122024-02-15T02:58:35.886Z 21 420-439 29 29 29 56 56 56 F F F
33132024-02-15T02:58:35.886Z 22 440-459 27 27 27 53 53 53 F F F
33142024-02-15T02:58:35.886Z 23 460-479 29 29 29 56 56 56 F F F
33152024-02-15T02:58:35.886Z 24 480-499 29 29 29 56 56 56 F F F
33162024-02-15T02:58:35.886Z 25 500-519 26 26 26 52 52 52 F F F
33172024-02-15T02:58:35.886Z 26 520-539 29 29 29 56 56 56 F F F
33182024-02-15T02:58:35.886Z 27 540-559 26 26 26 51 51 51 F F F
33192024-02-15T02:58:35.887Z 28 560-579 24 24 24 47 47 47 F F F
33202024-02-15T02:58:35.887Z 29 580-599 26 26 26 50 50 50 F F F
33212024-02-15T02:58:35.887ZMax gen: 29, Max flush: 56
33222024-02-15T02:58:35.887Z
33232024-02-15T02:58:35.887ZBegin loop 15
33242024-02-15T02:58:35.887ZDownstairs to restart: 2
33252024-02-15T02:58:35.887Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.818499336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255}
33262024-02-15T02:58:35.887Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.818779941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255}
33272024-02-15T02:58:35.887Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.818797788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255}
33282024-02-15T02:58:35.887Z{"msg":"Crucible 40255335-4e4d-4b0e-9866-42628f690ef4 has session id: 26d23fe6-095e-4c7e-9d56-334eb674880c","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.818814337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33292024-02-15T02:58:35.887Z{"msg":"Crucible runtime is spawned
33302024-02-15T02:58:35.887ZUpstairs opts: Upstairs UUID: 40255335-4e4d-4b0e-9866-42628f690ef4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.81886675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33312024-02-15T02:58:35.887Z{"msg":"Crucible stats registered with UUID: 40255335-4e4d-4b0e-9866-42628f690ef4","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.818915134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33322024-02-15T02:58:35.888Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.818938971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255}
33332024-02-15T02:58:35.888Z{"msg":"40255335-4e4d-4b0e-9866-42628f690ef4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819060846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33342024-02-15T02:58:35.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819079773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33352024-02-15T02:58:35.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819095381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33362024-02-15T02:58:35.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819132777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33372024-02-15T02:58:35.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819155114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33382024-02-15T02:58:35.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.8191873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33392024-02-15T02:58:35.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819212796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33402024-02-15T02:58:35.888Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819249042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33412024-02-15T02:58:35.888Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819295396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33422024-02-15T02:58:35.888Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819335391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33432024-02-15T02:58:35.888Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:34.819373386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33442024-02-15T02:58:35.889Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:34.819401893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33452024-02-15T02:58:35.889Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:34.819442568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33462024-02-15T02:58:35.889Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819473524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33472024-02-15T02:58:35.889Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.81950014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33482024-02-15T02:58:35.889Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819526547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33492024-02-15T02:58:35.889Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.819551274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33502024-02-15T02:58:35.889Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.81966159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33512024-02-15T02:58:35.889Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.820216561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33522024-02-15T02:58:35.890Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.820257276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33532024-02-15T02:58:35.890Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.820273763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33542024-02-15T02:58:35.890Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.82029896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33552024-02-15T02:58:35.890Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.820804417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33562024-02-15T02:58:35.890Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.820915773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33572024-02-15T02:58:35.891Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.821326362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33582024-02-15T02:58:35.891Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.821368756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33592024-02-15T02:58:35.891Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.821415511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33602024-02-15T02:58:35.891Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.821462365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33612024-02-15T02:58:35.891Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.821480382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33622024-02-15T02:58:35.891Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:34.82150194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33632024-02-15T02:58:36.911Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.821232896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33642024-02-15T02:58:36.911Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.821344142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33652024-02-15T02:58:36.930Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.822208594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33662024-02-15T02:58:36.930Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.82224467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33672024-02-15T02:58:36.930Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.822785992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33682024-02-15T02:58:36.930Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823353011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33692024-02-15T02:58:36.930Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823392626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33702024-02-15T02:58:36.931Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 40, 42, 54, 54, 49, 45, 45, 56, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823422802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33712024-02-15T02:58:36.931Z{"msg":"[0]R generation[0..12]: [14, 27, 22, 22, 28, 28, 25, 24, 24, 29, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823460937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33722024-02-15T02:58:36.931Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823477305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33732024-02-15T02:58:36.931Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 40, 42, 54, 54, 49, 45, 45, 56, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823499582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33742024-02-15T02:58:36.931Z{"msg":"[1]R generation[0..12]: [14, 27, 22, 22, 28, 28, 25, 24, 24, 29, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.82351387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33752024-02-15T02:58:36.931Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823528789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33762024-02-15T02:58:36.931Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 40, 42, 54, 54, 49, 45, 45, 56, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823549276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33772024-02-15T02:58:36.931Z{"msg":"[2]R generation[0..12]: [14, 27, 22, 22, 28, 28, 25, 24, 24, 29, 24, 26]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
33782024-02-15T02:58:36.931Z,"time":"2024-02-15T02:58:35.823563544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33792024-02-15T02:58:36.931Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823581482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33802024-02-15T02:58:36.932Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823601669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33812024-02-15T02:58:36.932Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823615918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33822024-02-15T02:58:36.932Z{"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823635585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33832024-02-15T02:58:36.932Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823649803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33842024-02-15T02:58:36.932Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823664101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33852024-02-15T02:58:36.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.82367641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33862024-02-15T02:58:36.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823696397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33872024-02-15T02:58:36.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823709976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33882024-02-15T02:58:36.932Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823730023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33892024-02-15T02:58:36.932Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823742952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33902024-02-15T02:58:36.933Z{"msg":"40255335-4e4d-4b0e-9866-42628f690ef4 is now active with session: 26d23fe6-095e-4c7e-9d56-334eb674880c","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.82375607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33912024-02-15T02:58:36.933Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823775787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255,"session_id":"26d23fe6-095e-4c7e-9d56-334eb674880c"}
33922024-02-15T02:58:36.933Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:35.823788306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255}
33932024-02-15T02:58:36.933ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33942024-02-15T02:58:36.933ZLoading write count information from file "/tmp/test_repair_verify.data"
33952024-02-15T02:58:36.933ZRun Repair workload
33962024-02-15T02:58:36.934Z01/30 Write block 501 len 24576 data: 5 5 5 5 5 4
33972024-02-15T02:58:36.934Z02/30 Read block 281 len 28672
33982024-02-15T02:58:36.934Z03/30 Read block 145 len 32768
33992024-02-15T02:58:36.934Z04/30 Read block 389 len 40960
34002024-02-15T02:58:36.934Z05/30 Read block 87 len 24576
34012024-02-15T02:58:36.934Z06/30 Write block 279 len 36864 data: 2 2 2 2 2 3 3 3 3
34022024-02-15T02:58:36.934Z07/30 Read block 415 len 32768
34032024-02-15T02:58:36.934Z08/30 Write block 185 len 36864 data: 5 5 3 3 4 3 3 4 6
34042024-02-15T02:58:36.934Z09/30 Flush
34052024-02-15T02:58:36.934Z10/30 Write block 444 len 32768 data: 4 5 5 5 4 2 2 2
34062024-02-15T02:58:36.935Z11/30 Write block 301 len 24576 data: 5 5 6 7 8 6
34072024-02-15T02:58:36.935Z12/30 Read block 285 len 24576
34082024-02-15T02:58:36.935Z13/30 Flush
34092024-02-15T02:58:36.935Z14/30 Write block 76 len 24576 data: 3 3 2 2 2 4
34102024-02-15T02:58:36.935Z15/30 Read block 336 len 36864
34112024-02-15T02:58:36.935Z16/30 Read block 426 len 12288
34122024-02-15T02:58:36.935Z17/30 Read block 535 len 32768
34132024-02-15T02:58:36.935Z18/30 Read block 161 len 4096
34142024-02-15T02:58:36.935Z19/30 Write block 98 len 20480 data: 8 6 5 6 6
34152024-02-15T02:58:36.935Z20/30 Read block 139 len 12288
34162024-02-15T02:58:36.935Z21/30 Read block 65 len 36864
34172024-02-15T02:58:36.936Z22/30 Read block 310 len 4096
34182024-02-15T02:58:36.936Z23/30 Read block 372 len 28672
34192024-02-15T02:58:36.936Z24/30 Read block 34 len 32768
34202024-02-15T02:58:36.936Z25/30 Read block 257 len 32768
34212024-02-15T02:58:36.936Z26/30 Read block 48 len 36864
34222024-02-15T02:58:36.936Z27/30 Read block 155 len 32768
34232024-02-15T02:58:36.937Z28/30 Write block 345 len 32768 data: 3 3 4 4 4 4 4 5
34242024-02-15T02:58:36.937Z29/30 Write block 541 len 16384 data: 4 6 5 4
34252024-02-15T02:58:36.937Z30/30 Read block 49 len 20480
34262024-02-15T02:58:36.937Z----------------------------------------------------------------
34272024-02-15T02:58:36.937Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
34282024-02-15T02:58:36.937ZStates: Active Active Active
34292024-02-15T02:58:36.937ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
34302024-02-15T02:58:36.937Z 1 Acked 1000 Write 6 Done Done Done false
34312024-02-15T02:58:36.937Z 2 Acked 1001 Read 7 Done Done Done false
34322024-02-15T02:58:36.937Z 3 Acked 1002 Read 8 Done Done Done false
34332024-02-15T02:58:36.938Z 4 Acked 1003 Read 10 Done Done Done false
34342024-02-15T02:58:36.938Z 5 Acked 1004 Read 6 Done Done Done false
34352024-02-15T02:58:36.938Z 6 Acked 1005 Write 9 Done Done Done false
34362024-02-15T02:58:36.938Z 7 Acked 1006 Read 8 Done Done Sent false
34372024-02-15T02:58:36.938Z 8 Acked 1007 Write 9 Done Done Sent false
34382024-02-15T02:58:36.938Z 9 Acked 1008 Flush 0 Done Done Sent false
34392024-02-15T02:58:36.938Z 10 Acked 1009 Write 8 Done Done Sent false
34402024-02-15T02:58:36.938Z 11 Acked 1010 Write 6 Done Done Sent false
34412024-02-15T02:58:36.938Z 12 Acked 1011 Read 6 Done Done Sent false
34422024-02-15T02:58:36.938Z 13 Acked 1012 Flush 0 Done Done Sent false
34432024-02-15T02:58:36.938Z 14 Acked 1013 Write 6 Done Done Sent false
34442024-02-15T02:58:36.939Z 15 Acked 1014 Read 9 Done Done Sent false
34452024-02-15T02:58:36.939Z 16 Acked 1015 Read 3 Done Done Sent false
34462024-02-15T02:58:36.939Z 17 Acked 1016 Read 8 Done Done Sent false
34472024-02-15T02:58:36.939Z 18 Acked 1017 Read 1 Done Done Sent false
34482024-02-15T02:58:36.939Z 19 Acked 1018 Write 5 Done Done Sent false
34492024-02-15T02:58:36.939Z 20 Acked 1019 Read 3 Done Done Sent false
34502024-02-15T02:58:36.939Z 21 Acked 1020 Read 9 Done Done Sent false
34512024-02-15T02:58:36.940Z 22 Acked 1021 Read 1 Done Done Sent false
34522024-02-15T02:58:36.940Z 23 Acked 1022 Read 7 Done Done Sent false
34532024-02-15T02:58:36.940Z 24 Acked 1023 Read 8 Done Done Sent false
34542024-02-15T02:58:36.940Z 25 Acked 1024 Read 8 Done Done Sent false
34552024-02-15T02:58:36.940Z 26 Acked 1025 Read 9 Done Done Sent false
34562024-02-15T02:58:36.940Z 27 Acked 1026 Read 8 Done Done Sent false
34572024-02-15T02:58:36.940Z 28 Acked 1027 Write 8 Done Done Sent false
34582024-02-15T02:58:36.940Z 29 Acked 1028 Write 4 Done Done Sent false
34592024-02-15T02:58:36.940Z 30 Acked 1029 Read 5 Done Sent Sent false
34602024-02-15T02:58:36.940Z STATES DS:0 DS:1 DS:2 TOTAL
34612024-02-15T02:58:36.941Z New 0 0 0 0
34622024-02-15T02:58:36.941Z Sent 0 1 24 25
34632024-02-15T02:58:36.941Z Done 30 29 6 65
34642024-02-15T02:58:36.941Z Skipped 0 0 0 0
34652024-02-15T02:58:36.957Z Error 0 0 0 0
34662024-02-15T02:58:36.957ZLast Flush: 1012 1012 0
34672024-02-15T02:58:36.957ZDownstairs last five completed:
34682024-02-15T02:58:36.957ZUpstairs last five completed: 30 29 28 27 26
34692024-02-15T02:58:36.957Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:35.856727129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1255}
34702024-02-15T02:58:36.957ZWrote out file "/tmp/test_repair_verify.data"
34712024-02-15T02:58:36.957Z
34722024-02-15T02:58:36.958ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
34732024-02-15T02:58:36.958ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34742024-02-15T02:58:36.958Z 0 000-019 14 14 14 25 25 25 F F F
34752024-02-15T02:58:36.958Z 1 020-039 27 27 27 53 53 53 F F F
34762024-02-15T02:58:36.958Z 2 040-059 22 22 22 40 40 40 F F F
34772024-02-15T02:58:36.958Z 3 060-079 22 22 22 42 42 42 T T F <---
34782024-02-15T02:58:36.958Z 4 080-099 28 28 28 54 54 54 T T F <---
34792024-02-15T02:58:36.958Z 5 100-119 28 28 28 54 54 54 T T F <---
34802024-02-15T02:58:36.958Z 6 120-139 25 25 25 49 49 49 F F F
34812024-02-15T02:58:36.958Z 7 140-159 24 24 24 45 45 45 F F F
34822024-02-15T02:58:36.958Z 8 160-179 24 24 24 45 45 45 F F F
34832024-02-15T02:58:36.958Z 9 180-199 30 30 29 57 57 56 F F F <---
34842024-02-15T02:58:36.958Z 10 200-219 24 24 24 45 45 45 F F F
34852024-02-15T02:58:36.958Z 11 220-239 26 26 26 51 51 51 F F F
34862024-02-15T02:58:36.958Z 12 240-259 28 28 28 54 54 54 F F F
34872024-02-15T02:58:36.958Z 13 260-279 30 30 28 57 57 54 F F T <---
34882024-02-15T02:58:36.958Z 14 280-299 30 30 27 57 57 53 F F T <---
34892024-02-15T02:58:36.958Z 15 300-319 30 30 27 58 58 53 F F F <---
34902024-02-15T02:58:36.958Z 16 320-339 26 26 26 52 52 52 F F F
34912024-02-15T02:58:36.958Z 17 340-359 28 28 28 55 55 55 T T F <---
34922024-02-15T02:58:36.958Z 18 360-379 28 28 28 55 55 55 F F F
34932024-02-15T02:58:36.958Z 19 380-399 29 29 29 56 56 56 F F F
34942024-02-15T02:58:36.958Z 20 400-419 24 24 24 45 45 45 F F F
34952024-02-15T02:58:36.958Z 21 420-439 29 29 29 56 56 56 F F F
34962024-02-15T02:58:36.958Z 22 440-459 30 30 27 58 58 53 F F F <---
34972024-02-15T02:58:36.959Z 23 460-479 29 29 29 56 56 56 F F F
34982024-02-15T02:58:36.959Z 24 480-499 29 29 29 56 56 56 F F F
34992024-02-15T02:58:36.959Z 25 500-519 30 30 26 57 57 52 F F T <---
35002024-02-15T02:58:36.959Z 26 520-539 29 29 29 56 56 56 F F F
35012024-02-15T02:58:36.959Z 27 540-559 26 26 26 51 51 51 T T F <---
35022024-02-15T02:58:36.959Z 28 560-579 24 24 24 47 47 47 F F F
35032024-02-15T02:58:36.959Z 29 580-599 26 26 26 50 50 50 F F F
35042024-02-15T02:58:36.959ZMax gen: 30, Max flush: 58
35052024-02-15T02:58:36.959ZError: Difference in extent metadata found!
35062024-02-15T02:58:36.959ZA Difference in extent metadata is expected here
35072024-02-15T02:58:36.959ZOn loop 15
35082024-02-15T02:58:36.959Z
35092024-02-15T02:58:36.959ZVerifying data now
35102024-02-15T02:58:42.524ZNo change after verify
35112024-02-15T02:58:42.524ZLoop: 15 Downstairs dump after verify (and repair):
35122024-02-15T02:58:42.536ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35132024-02-15T02:58:42.536Z 0 000-019 14 14 14 25 25 25 F F F
35142024-02-15T02:58:42.536Z 1 020-039 27 27 27 53 53 53 F F F
35152024-02-15T02:58:42.536Z 2 040-059 22 22 22 40 40 40 F F F
35162024-02-15T02:58:42.536Z 3 060-079 31 31 31 59 59 59 F F F
35172024-02-15T02:58:42.536Z 4 080-099 31 31 31 59 59 59 F F F
35182024-02-15T02:58:42.536Z 5 100-119 31 31 31 59 59 59 F F F
35192024-02-15T02:58:42.537Z 6 120-139 25 25 25 49 49 49 F F F
35202024-02-15T02:58:42.537Z 7 140-159 24 24 24 45 45 45 F F F
35212024-02-15T02:58:42.537Z 8 160-179 24 24 24 45 45 45 F F F
35222024-02-15T02:58:42.537Z 9 180-199 30 30 30 57 57 57 F F F
35232024-02-15T02:58:42.537Z 10 200-219 24 24 24 45 45 45 F F F
35242024-02-15T02:58:42.537Z 11 220-239 26 26 26 51 51 51 F F F
35252024-02-15T02:58:42.537Z 12 240-259 28 28 28 54 54 54 F F F
35262024-02-15T02:58:42.537Z 13 260-279 30 30 30 57 57 57 F F F
35272024-02-15T02:58:42.537Z 14 280-299 30 30 30 57 57 57 F F F
35282024-02-15T02:58:42.537Z 15 300-319 30 30 30 58 58 58 F F F
35292024-02-15T02:58:42.537Z 16 320-339 26 26 26 52 52 52 F F F
35302024-02-15T02:58:42.537Z 17 340-359 31 31 31 59 59 59 F F F
35312024-02-15T02:58:42.537Z 18 360-379 28 28 28 55 55 55 F F F
35322024-02-15T02:58:42.537Z 19 380-399 29 29 29 56 56 56 F F F
35332024-02-15T02:58:42.537Z 20 400-419 24 24 24 45 45 45 F F F
35342024-02-15T02:58:42.537Z 21 420-439 29 29 29 56 56 56 F F F
35352024-02-15T02:58:42.538Z 22 440-459 30 30 30 58 58 58 F F F
35362024-02-15T02:58:42.538Z 23 460-479 29 29 29 56 56 56 F F F
35372024-02-15T02:58:42.538Z 24 480-499 29 29 29 56 56 56 F F F
35382024-02-15T02:58:42.538Z 25 500-519 30 30 30 57 57 57 F F F
35392024-02-15T02:58:42.538Z 26 520-539 29 29 29 56 56 56 F F F
35402024-02-15T02:58:42.538Z 27 540-559 31 31 31 59 59 59 F F F
35412024-02-15T02:58:42.538Z 28 560-579 24 24 24 47 47 47 F F F
35422024-02-15T02:58:42.538Z 29 580-599 26 26 26 50 50 50 F F F
35432024-02-15T02:58:42.538ZMax gen: 31, Max flush: 59
35442024-02-15T02:58:42.544Z
35452024-02-15T02:58:42.544ZBegin loop 16
35462024-02-15T02:58:42.544ZDownstairs to restart: 1
35472024-02-15T02:58:42.558Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.498606948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263}
35482024-02-15T02:58:42.558Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.498906291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263}
35492024-02-15T02:58:42.558Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.498939327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263}
35502024-02-15T02:58:42.570Z{"msg":"Crucible aae42163-42da-4b91-8fca-7ad59877e01d has session id: dc8a030f-dd61-42fc-8021-5176c0a7a5dc","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.498969573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35512024-02-15T02:58:42.570Z{"msg":"Upstairs opts: Upstairs UUID: aae42163-42da-4b91-8fca-7ad59877e01d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499011128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35522024-02-15T02:58:42.570Z{"msg":"Crucible stats registered with UUID: aae42163-42da-4b91-8fca-7ad59877e01d","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499036305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35532024-02-15T02:58:42.570ZCrucible runtime is spawned
35542024-02-15T02:58:42.570Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499159289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263}
35552024-02-15T02:58:42.571Z{"msg":"aae42163-42da-4b91-8fca-7ad59877e01d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499294472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35562024-02-15T02:58:42.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499324669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35572024-02-15T02:58:42.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499372403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35582024-02-15T02:58:42.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499407478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35592024-02-15T02:58:42.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499435565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35602024-02-15T02:58:42.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.49947335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35612024-02-15T02:58:42.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499503686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35622024-02-15T02:58:42.571Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499537512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35632024-02-15T02:58:42.571Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499564648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35642024-02-15T02:58:42.571Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499596295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35652024-02-15T02:58:42.571Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:41.49971588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35662024-02-15T02:58:42.571Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:41.499771013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35672024-02-15T02:58:42.571Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499799959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35682024-02-15T02:58:42.571Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499844244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35692024-02-15T02:58:42.571Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:41.499887158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35702024-02-15T02:58:42.572Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499926163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35712024-02-15T02:58:42.572Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.499974817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35722024-02-15T02:58:42.572Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.500012692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35732024-02-15T02:58:42.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.500504471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35742024-02-15T02:58:42.572Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.500542806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35752024-02-15T02:58:42.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.500559684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35762024-02-15T02:58:42.572Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.50058679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35772024-02-15T02:58:42.572Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.501134992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35782024-02-15T02:58:42.572Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.501289642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35792024-02-15T02:58:42.572Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.501670275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35802024-02-15T02:58:42.572Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.501717669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35812024-02-15T02:58:42.572Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.501745575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35822024-02-15T02:58:42.572Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.50187041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35832024-02-15T02:58:42.573Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.501907375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35842024-02-15T02:58:42.573Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:41.501930072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35852024-02-15T02:58:43.567Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.501603846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35862024-02-15T02:58:43.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.501756017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35872024-02-15T02:58:43.583Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.50261504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35882024-02-15T02:58:43.584Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.502655554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35892024-02-15T02:58:43.584Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.50316916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35902024-02-15T02:58:43.584Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.50373214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35912024-02-15T02:58:43.584Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.503771595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35922024-02-15T02:58:43.584Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 40, 59, 59, 59, 49, 45, 45, 57, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.503799411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35932024-02-15T02:58:43.584Z{"msg":"[0]R generation[0..12]: [14, 27, 22, 31, 31, 31, 25, 24, 24, 30, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.503834827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35942024-02-15T02:58:43.585Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.503860324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35952024-02-15T02:58:43.585Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 40, 59, 59, 59, 49, 45, 45, 57, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.503894119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35962024-02-15T02:58:43.585Z{"msg":"[1]R generation[0..12]: [14, 27, 22, 31, 31, 31, 25, 24, 24, 30, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.503913307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35972024-02-15T02:58:43.585Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.503945503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
35982024-02-15T02:58:43.586Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 40, 59, 59, 59, 49, 45, 45, 57, 45, 51]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
35992024-02-15T02:58:43.586Zcrucible","level":30,"time":"2024-02-15T02:58:42.50396575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36002024-02-15T02:58:43.586Z{"msg":"[2]R generation[0..12]: [14, 27, 22, 31, 31, 31, 25, 24, 24, 30, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504010055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36012024-02-15T02:58:43.586Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504035811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36022024-02-15T02:58:43.586Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504054199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36032024-02-15T02:58:43.586Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504068267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36042024-02-15T02:58:43.587Z{"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504088905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36052024-02-15T02:58:43.587Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504110192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36062024-02-15T02:58:43.587Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504131599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36072024-02-15T02:58:43.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504162215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36082024-02-15T02:58:43.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504188242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36092024-02-15T02:58:43.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504220518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36102024-02-15T02:58:43.588Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504245395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36112024-02-15T02:58:43.588Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504269642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36122024-02-15T02:58:43.588Z{"msg":"aae42163-42da-4b91-8fca-7ad59877e01d is now active with session: dc8a030f-dd61-42fc-8021-5176c0a7a5dc","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.50429083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36132024-02-15T02:58:43.589Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504323835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263,"session_id":"dc8a030f-dd61-42fc-8021-5176c0a7a5dc"}
36142024-02-15T02:58:43.589Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:42.504357661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263}
36152024-02-15T02:58:43.589ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
36162024-02-15T02:58:43.589ZLoading write count information from file "/tmp/test_repair_verify.data"
36172024-02-15T02:58:43.589ZRun Repair workload
36182024-02-15T02:58:43.589Z01/30 Write block 136 len 4096 data: 5
36192024-02-15T02:58:43.590Z02/30 Read block 536 len 28672
36202024-02-15T02:58:43.590Z03/30 Write block 506 len 8192 data: 5 3
36212024-02-15T02:58:43.590Z04/30 Read block 208 len 12288
36222024-02-15T02:58:43.590Z05/30 Flush
36232024-02-15T02:58:43.590Z06/30 Write block 392 len 8192 data: 3 3
36242024-02-15T02:58:43.590Z07/30 Flush
36252024-02-15T02:58:43.591Z08/30 Write block 445 len 20480 data: 6 6 6 5 3
36262024-02-15T02:58:43.591Z09/30 Write block 167 len 36864 data: 4 4 4 4 4 3 3 2 3
36272024-02-15T02:58:43.591Z10/30 Write block 153 len 12288 data: 4 3 2
36282024-02-15T02:58:43.591Z11/30 Read block 34 len 16384
36292024-02-15T02:58:43.591Z12/30 Write block 305 len 36864 data: 9 7 6 5 6 7 6 5 5
36302024-02-15T02:58:43.591Z13/30 Write block 534 len 16384 data: 3 4 4 4
36312024-02-15T02:58:43.591Z14/30 Read block 433 len 20480
36322024-02-15T02:58:43.591Z15/30 Write block 268 len 8192 data: 3 3
36332024-02-15T02:58:43.591Z16/30 Write block 301 len 16384 data: 6 6 7 8
36342024-02-15T02:58:43.591Z17/30 Write block 241 len 36864 data: 4 5 5 5 5 4 3 3 3
36352024-02-15T02:58:43.602Z18/30 Read block 361 len 36864
36362024-02-15T02:58:43.602Z19/30 Read block 374 len 20480
36372024-02-15T02:58:43.602Z20/30 Read block 30 len 28672
36382024-02-15T02:58:43.602Z21/30 Write block 127 len 36864 data: 5 4 4 3 3 4 5 5 5
36392024-02-15T02:58:43.603Z22/30 Write block 571 len 4096 data: 2
36402024-02-15T02:58:43.603Z23/30 Write block 119 len 20480 data: 3 3 4 3 3
36412024-02-15T02:58:43.603Z24/30 Read block 263 len 28672
36422024-02-15T02:58:43.603Z25/30 Read block 146 len 32768
36432024-02-15T02:58:43.603Z26/30 Flush
36442024-02-15T02:58:43.604Z27/30 Write block 343 len 36864 data: 2 3 4 4 5 5 5 5 5
36452024-02-15T02:58:43.604Z28/30 Write block 570 len 36864 data: 2 3 2 4 4 5 5 4 4
36462024-02-15T02:58:43.604Z29/30 Write block 465 len 32768 data: 2 2 2 3 4 5 5 4
36472024-02-15T02:58:43.604Z30/30 Read block 251 len 8192
36482024-02-15T02:58:43.604Z----------------------------------------------------------------
36492024-02-15T02:58:43.604Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
36502024-02-15T02:58:43.605ZStates: Active Active Active
36512024-02-15T02:58:43.605ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
36522024-02-15T02:58:43.605Z 1 Acked 1000 Write 1 Done Sent Done false
36532024-02-15T02:58:43.605Z 2 Acked 1001 Read 7 Done Sent Done false
36542024-02-15T02:58:43.605Z 3 Acked 1002 Write 2 Done Sent Done false
36552024-02-15T02:58:43.605Z 4 Acked 1003 Read 3 Done Sent Done false
36562024-02-15T02:58:43.605Z 5 Acked 1004 Flush 0 Done Sent Done false
36572024-02-15T02:58:43.606Z 6 Acked 1005 Write 2 Done Sent Done false
36582024-02-15T02:58:43.606Z 7 Acked 1006 Flush 0 Done Sent Done false
36592024-02-15T02:58:43.606Z 8 Acked 1007 Write 5 Done Sent Done false
36602024-02-15T02:58:43.606Z 9 Acked 1008 Write 9 Done Sent Done false
36612024-02-15T02:58:43.606Z 10 Acked 1009 Write 3 Done Sent Done false
36622024-02-15T02:58:43.606Z 11 Acked 1010 Read 4 Done Sent Done false
36632024-02-15T02:58:43.606Z 12 Acked 1011 Write 9 Done Sent Done false
36642024-02-15T02:58:43.606Z 13 Acked 1012 Write 4 Done Sent Done false
36652024-02-15T02:58:43.606Z 14 Acked 1013 Read 5 Done Sent Done false
36662024-02-15T02:58:43.606Z 15 Acked 1014 Write 2 Done Sent Done false
36672024-02-15T02:58:43.607Z 16 Acked 1015 Write 4 Done Sent Done false
36682024-02-15T02:58:43.607Z 17 Acked 1016 Write 9 Done Sent Done false
36692024-02-15T02:58:43.607Z 18 Acked 1017 Read 9 Done Sent Done false
36702024-02-15T02:58:43.607Z 19 Acked 1018 Read 5 Done Sent Done false
36712024-02-15T02:58:43.607Z 20 Acked 1019 Read 7 Done Sent Done false
36722024-02-15T02:58:43.608Z 21 Acked 1020 Write 9 Done Sent Done false
36732024-02-15T02:58:43.608Z 22 Acked 1021 Write 1 Done Sent Done false
36742024-02-15T02:58:43.608Z 23 Acked 1022 Write 5 Done Sent Done false
36752024-02-15T02:58:43.608Z 24 Acked 1023 Read 7 Done Sent Done false
36762024-02-15T02:58:43.609Z 25 Acked 1024 Read 8 Done Sent Done false
36772024-02-15T02:58:43.609Z 26 Acked 1025 Flush 0 Done Sent Done false
36782024-02-15T02:58:43.609Z 27 Acked 1026 Write 9 Done Sent Done false
36792024-02-15T02:58:43.609Z 28 Acked 1027 Write 9 Done Sent Done false
36802024-02-15T02:58:43.610Z 29 Acked 1028 Write 8 Done Sent Done false
36812024-02-15T02:58:43.610Z 30 Acked 1029 Read 2 Done Sent Done false
36822024-02-15T02:58:43.610Z STATES DS:0 DS:1 DS:2 TOTAL
36832024-02-15T02:58:43.610Z New 0 0 0 0
36842024-02-15T02:58:43.610Z Sent 0 30 0 30
36852024-02-15T02:58:43.610Z Done 30 0 30 60
36862024-02-15T02:58:43.610Z Skipped 0 0 0 0
36872024-02-15T02:58:43.611Z Error 0 0 0 0
36882024-02-15T02:58:43.611ZLast Flush: 1025 0 1025
36892024-02-15T02:58:43.611ZDownstairs last five completed:
36902024-02-15T02:58:43.611ZUpstairs last five completed: 30 29 28 27 26
36912024-02-15T02:58:43.611Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:42.53812684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1263}
36922024-02-15T02:58:43.611ZWrote out file "/tmp/test_repair_verify.data"
36932024-02-15T02:58:43.611Z
36942024-02-15T02:58:43.642ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36952024-02-15T02:58:43.649ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36962024-02-15T02:58:43.649Z 0 000-019 14 14 14 25 25 25 F F F
36972024-02-15T02:58:43.649Z 1 020-039 27 27 27 53 53 53 F F F
36982024-02-15T02:58:43.649Z 2 040-059 22 22 22 40 40 40 F F F
36992024-02-15T02:58:43.649Z 3 060-079 31 31 31 59 59 59 F F F
37002024-02-15T02:58:43.650Z 4 080-099 31 31 31 59 59 59 F F F
37012024-02-15T02:58:43.650Z 5 100-119 32 31 32 62 59 62 F F F <---
37022024-02-15T02:58:43.650Z 6 120-139 32 25 32 62 49 62 F F F <---
37032024-02-15T02:58:43.650Z 7 140-159 32 24 32 62 45 62 F F F <---
37042024-02-15T02:58:43.650Z 8 160-179 32 24 32 62 45 62 F F F <---
37052024-02-15T02:58:43.650Z 9 180-199 30 30 30 57 57 57 F F F
37062024-02-15T02:58:43.650Z 10 200-219 24 24 24 45 45 45 F F F
37072024-02-15T02:58:43.650Z 11 220-239 26 26 26 51 51 51 F F F
37082024-02-15T02:58:43.650Z 12 240-259 32 28 32 62 54 62 F F F <---
37092024-02-15T02:58:43.650Z 13 260-279 32 30 32 62 57 62 F F F <---
37102024-02-15T02:58:43.650Z 14 280-299 30 30 30 57 57 57 F F F
37112024-02-15T02:58:43.650Z 15 300-319 32 30 32 62 58 62 F F F <---
37122024-02-15T02:58:43.650Z 16 320-339 26 26 26 52 52 52 F F F
37132024-02-15T02:58:43.651Z 17 340-359 31 31 31 59 59 59 T F T <---
37142024-02-15T02:58:43.651Z 18 360-379 28 28 28 55 55 55 F F F
37152024-02-15T02:58:43.651Z 19 380-399 32 29 32 61 56 61 F F F <---
37162024-02-15T02:58:43.651Z 20 400-419 24 24 24 45 45 45 F F F
37172024-02-15T02:58:43.651Z 21 420-439 29 29 29 56 56 56 F F F
37182024-02-15T02:58:43.651Z 22 440-459 32 30 32 62 58 62 F F F <---
37192024-02-15T02:58:43.651Z 23 460-479 29 29 29 56 56 56 T F T <---
37202024-02-15T02:58:43.651Z 24 480-499 29 29 29 56 56 56 F F F
37212024-02-15T02:58:43.651Z 25 500-519 32 30 32 60 57 60 F F F <---
37222024-02-15T02:58:43.651Z 26 520-539 32 29 32 62 56 62 F F F <---
37232024-02-15T02:58:43.651Z 27 540-559 31 31 31 59 59 59 F F F
37242024-02-15T02:58:43.651Z 28 560-579 32 24 32 62 47 62 T F T <---
37252024-02-15T02:58:43.651Z 29 580-599 26 26 26 50 50 50 F F F
37262024-02-15T02:58:43.652ZMax gen: 32, Max flush: 62
37272024-02-15T02:58:43.652ZError: Difference in extent metadata found!
37282024-02-15T02:58:43.652ZA Difference in extent metadata is expected here
37292024-02-15T02:58:43.652ZOn loop 16
37302024-02-15T02:58:43.652Z
37312024-02-15T02:58:43.652ZVerifying data now
37322024-02-15T02:58:48.333ZNo change after verify
37332024-02-15T02:58:48.334ZLoop: 16 Downstairs dump after verify (and repair):
37342024-02-15T02:58:48.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
37352024-02-15T02:58:48.351Z 0 000-019 14 14 14 25 25 25 F F F
37362024-02-15T02:58:48.351Z 1 020-039 27 27 27 53 53 53 F F F
37372024-02-15T02:58:48.351Z 2 040-059 22 22 22 40 40 40 F F F
37382024-02-15T02:58:48.352Z 3 060-079 31 31 31 59 59 59 F F F
37392024-02-15T02:58:48.352Z 4 080-099 31 31 31 59 59 59 F F F
37402024-02-15T02:58:48.353Z 5 100-119 32 32 32 62 62 62 F F F
37412024-02-15T02:58:48.353Z 6 120-139 32 32 32 62 62 62 F F F
37422024-02-15T02:58:48.353Z 7 140-159 32 32 32 62 62 62 F F F
37432024-02-15T02:58:48.354Z 8 160-179 32 32 32 62 62 62 F F F
37442024-02-15T02:58:48.354Z 9 180-199 30 30 30 57 57 57 F F F
37452024-02-15T02:58:48.354Z 10 200-219 24 24 24 45 45 45 F F F
37462024-02-15T02:58:48.354Z 11 220-239 26 26 26 51 51 51 F F F
37472024-02-15T02:58:48.355Z 12 240-259 32 32 32 62 62 62 F F F
37482024-02-15T02:58:48.355Z 13 260-279 32 32 32 62 62 62 F F F
37492024-02-15T02:58:48.355Z 14 280-299 30 30 30 57 57 57 F F F
37502024-02-15T02:58:48.355Z 15 300-319 32 32 32 62 62 62 F F F
37512024-02-15T02:58:48.356Z 16 320-339 26 26 26 52 52 52 F F F
37522024-02-15T02:58:48.356Z 17 340-359 33 33 33 63 63 63 F F F
37532024-02-15T02:58:48.356Z 18 360-379 28 28 28 55 55 55 F F F
37542024-02-15T02:58:48.356Z 19 380-399 32 32 32 61 61 61 F F F
37552024-02-15T02:58:48.356Z 20 400-419 24 24 24 45 45 45 F F F
37562024-02-15T02:58:48.357Z 21 420-439 29 29 29 56 56 56 F F F
37572024-02-15T02:58:48.357Z 22 440-459 32 32 32 62 62 62 F F F
37582024-02-15T02:58:48.357Z 23 460-479 33 33 33 63 63 63 F F F
37592024-02-15T02:58:48.357Z 24 480-499 29 29 29 56 56 56 F F F
37602024-02-15T02:58:48.357Z 25 500-519 32 32 32 60 60 60 F F F
37612024-02-15T02:58:48.358Z 26 520-539 32 32 32 62 62 62 F F F
37622024-02-15T02:58:48.358Z 27 540-559 31 31 31 59 59 59 F F F
37632024-02-15T02:58:48.358Z 28 560-579 33 33 33 63 63 63 F F F
37642024-02-15T02:58:48.358Z 29 580-599 26 26 26 50 50 50 F F F
37652024-02-15T02:58:48.358ZMax gen: 33, Max flush: 63
37662024-02-15T02:58:48.372Z
37672024-02-15T02:58:48.372ZBegin loop 17
37682024-02-15T02:58:48.372ZDownstairs to restart: 1
37692024-02-15T02:58:48.405Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312338774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271}
37702024-02-15T02:58:48.406Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312623199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271}
37712024-02-15T02:58:48.406Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312662194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271}
37722024-02-15T02:58:48.406Z{"msg":"Crucible ba261d4b-aba3-413c-b0d7-ce396abf4b9b has session id: b5eb059d-45ae-4c61-9df6-9a8b75a4dc52","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312685011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271Crucible runtime is spawned
37732024-02-15T02:58:48.406Z,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37742024-02-15T02:58:48.406Z{"msg":"Upstairs opts: Upstairs UUID: ba261d4b-aba3-413c-b0d7-ce396abf4b9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312755332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37752024-02-15T02:58:48.406Z{"msg":"Crucible stats registered with UUID: ba261d4b-aba3-413c-b0d7-ce396abf4b9b","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312795637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37762024-02-15T02:58:48.407Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312819774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271}
37772024-02-15T02:58:48.407Z{"msg":"ba261d4b-aba3-413c-b0d7-ce396abf4b9b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.31285161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37782024-02-15T02:58:48.407Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312891555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37792024-02-15T02:58:48.407Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312951488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37802024-02-15T02:58:48.407Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.312998492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37812024-02-15T02:58:48.408Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313038847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37822024-02-15T02:58:48.408Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313121066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37832024-02-15T02:58:48.408Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313192297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37842024-02-15T02:58:48.408Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313228383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37852024-02-15T02:58:48.408Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313259269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37862024-02-15T02:58:48.409Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313295824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37872024-02-15T02:58:48.409Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313339359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37882024-02-15T02:58:48.409Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313380524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37892024-02-15T02:58:48.409Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:47.313424458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37902024-02-15T02:58:48.409Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:47.313469003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37912024-02-15T02:58:48.409Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:47.313513437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37922024-02-15T02:58:48.410Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313539774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37932024-02-15T02:58:48.410Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313578719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37942024-02-15T02:58:48.410Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.313618914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37952024-02-15T02:58:48.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.314062328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37962024-02-15T02:58:48.410Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.314097224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37972024-02-15T02:58:48.410Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.314111682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37982024-02-15T02:58:48.410Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.314150348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
37992024-02-15T02:58:48.411Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.314680011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38002024-02-15T02:58:48.411Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.314818574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38012024-02-15T02:58:48.411Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.315222253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38022024-02-15T02:58:48.412Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.315268567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38032024-02-15T02:58:48.412Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.315339199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38042024-02-15T02:58:48.412Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.315376224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38052024-02-15T02:58:48.412Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.315392702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38062024-02-15T02:58:48.412Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:47.315413229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38072024-02-15T02:58:49.370Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.315210667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38082024-02-15T02:58:49.370Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.315306246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38092024-02-15T02:58:49.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.316215142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38102024-02-15T02:58:49.376Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.316259946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38112024-02-15T02:58:49.376Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.316755254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38122024-02-15T02:58:49.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317327742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38132024-02-15T02:58:49.376Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317364678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38142024-02-15T02:58:49.376Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 40, 59, 59, 62, 62, 62, 62, 57, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317384195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38152024-02-15T02:58:49.376Z{"msg":"[0]R generation[0..12]: [14, 27, 22, 31, 31, 32, 32, 32, 32, 30, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317406902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38162024-02-15T02:58:49.376Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317449907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38172024-02-15T02:58:49.376Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 40, 59, 59, 62, 62, 62, 62, 57, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317477843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38182024-02-15T02:58:49.376Z{"msg":"[1]R generation[0..12]: [14, 27, 22, 31, 31, 32, 32, 32, 32, 30, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317495701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38192024-02-15T02:58:49.376Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317521548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38202024-02-15T02:58:49.376Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 40, 59, 59, 62, 62, 62, 62, 57, 45, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317539276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38212024-02-15T02:58:49.376Z{"msg":"[2]R generation[0..12]: [14, 27, 22, 31, 31, 32, 32, 32, 32, 30, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317556404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38222024-02-15T02:58:49.376ZWait for a query_work_queue command to finish before sending IO
38232024-02-15T02:58:49.376Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317590579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38242024-02-15T02:58:49.376Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317616566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38252024-02-15T02:58:49.376Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317634024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38262024-02-15T02:58:49.376Z{"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317672139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38272024-02-15T02:58:49.377Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317697336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38282024-02-15T02:58:49.377Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317725163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38292024-02-15T02:58:49.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.31774059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38302024-02-15T02:58:49.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317765757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38312024-02-15T02:58:49.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317782405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38322024-02-15T02:58:49.377Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317807382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38332024-02-15T02:58:49.377Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.31782337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38342024-02-15T02:58:49.377Z{"msg":"ba261d4b-aba3-413c-b0d7-ce396abf4b9b is now active with session: b5eb059d-45ae-4c61-9df6-9a8b75a4dc52","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317848457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38352024-02-15T02:58:49.377Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317863685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271,"session_id":"b5eb059d-45ae-4c61-9df6-9a8b75a4dc52"}
38362024-02-15T02:58:49.377Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:48.317886922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271}
38372024-02-15T02:58:49.377ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
38382024-02-15T02:58:49.377ZLoading write count information from file "/tmp/test_repair_verify.data"
38392024-02-15T02:58:49.377ZRun Repair workload
38402024-02-15T02:58:49.381Z01/30 Write block 500 len 12288 data: 4 6 6
38412024-02-15T02:58:49.381Z02/30 Write block 294 len 24576 data: 3 2 3 3 3 4
38422024-02-15T02:58:49.381Z03/30 Read block 544 len 32768
38432024-02-15T02:58:49.381Z04/30 Write block 461 len 8192 data: 2 2
38442024-02-15T02:58:49.381Z05/30 Write block 236 len 8192 data: 4 3
38452024-02-15T02:58:49.381Z06/30 Read block 199 len 4096
38462024-02-15T02:58:49.381Z07/30 Write block 290 len 4096 data: 3
38472024-02-15T02:58:49.381Z08/30 Read block 26 len 12288
38482024-02-15T02:58:49.381Z09/30 Read block 7 len 32768
38492024-02-15T02:58:49.381Z10/30 Flush
38502024-02-15T02:58:49.385Z11/30 Write block 383 len 40960 data: 4 5 3 3 3 3 3 3 3 4
38512024-02-15T02:58:49.385Z12/30 Read block 323 len 16384
38522024-02-15T02:58:49.388Z13/30 Read block 397 len 8192
38532024-02-15T02:58:49.388Z14/30 Flush
38542024-02-15T02:58:49.391Z15/30 Write block 502 len 12288 data: 7 6 6
38552024-02-15T02:58:49.391Z16/30 Write block 406 len 28672 data: 3 4 4 5 5 5 6
38562024-02-15T02:58:49.391Z17/30 Read block 83 len 24576
38572024-02-15T02:58:49.391Z18/30 Write block 290 len 28672 data: 4 3 3 3 4 3 4
38582024-02-15T02:58:49.391Z19/30 Read block 316 len 4096
38592024-02-15T02:58:49.394Z20/30 Write block 245 len 40960 data: 6 5 4 4 4 3 2 3 3 4
38602024-02-15T02:58:49.394Z21/30 Write block 170 len 20480 data: 5 5 4 4 3
38612024-02-15T02:58:49.394Z22/30 Write block 229 len 32768 data: 5 4 3 4 4 4 4 5
38622024-02-15T02:58:49.395Z23/30 Flush
38632024-02-15T02:58:49.398Z24/30 Write block 583 len 16384 data: 2 2 2 3
38642024-02-15T02:58:49.398Z25/30 Read block 140 len 16384
38652024-02-15T02:58:49.401Z26/30 Flush
38662024-02-15T02:58:49.401Z27/30 Write block 53 len 24576 data: 2 2 2 2 3 3
38672024-02-15T02:58:49.405Z28/30 Read block 155 len 20480
38682024-02-15T02:58:49.405Z29/30 Read block 486 len 8192
38692024-02-15T02:58:49.405Z30/30 Write block 462 len 16384 data: 3 2 2 3
38702024-02-15T02:58:49.405Z----------------------------------------------------------------
38712024-02-15T02:58:49.405Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
38722024-02-15T02:58:49.406ZStates: Active Active Active
38732024-02-15T02:58:49.406ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
38742024-02-15T02:58:49.406Z 1 Acked 1000 Write 3 Done Done Done false
38752024-02-15T02:58:49.406Z 2 Acked 1001 Write 6 Done Sent Done false
38762024-02-15T02:58:49.406Z 3 Acked 1002 Read 8 Done Sent Done false
38772024-02-15T02:58:49.406Z 4 Acked 1003 Write 2 Done Sent Done false
38782024-02-15T02:58:49.406Z 5 Acked 1004 Write 2 Done Sent Done false
38792024-02-15T02:58:49.406Z 6 Acked 1005 Read 1 Done Sent Done false
38802024-02-15T02:58:49.406Z 7 Acked 1006 Write 1 Done Sent Done false
38812024-02-15T02:58:49.406Z 8 Acked 1007 Read 3 Done Sent Done false
38822024-02-15T02:58:49.406Z 9 Acked 1008 Read 8 Done Sent Done false
38832024-02-15T02:58:49.406Z 10 Acked 1009 Flush 0 Done Sent Done false
38842024-02-15T02:58:49.406Z 11 Acked 1010 Write 10 Done Sent Done false
38852024-02-15T02:58:49.406Z 12 Acked 1011 Read 4 Done Sent Done false
38862024-02-15T02:58:49.406Z 13 Acked 1012 Read 2 Done Sent Done false
38872024-02-15T02:58:49.407Z 14 Acked 1013 Flush 0 Done Sent Done false
38882024-02-15T02:58:49.407Z 15 Acked 1014 Write 3 Done Sent Done false
38892024-02-15T02:58:49.407Z 16 Acked 1015 Write 7 Done Sent Done false
38902024-02-15T02:58:49.407Z 17 Acked 1016 Read 6 Done Sent Done false
38912024-02-15T02:58:49.407Z 18 Acked 1017 Write 7 Done Sent Done false
38922024-02-15T02:58:49.407Z 19 Acked 1018 Read 1 Done Sent Done false
38932024-02-15T02:58:49.407Z 20 Acked 1019 Write 10 Done Sent Done false
38942024-02-15T02:58:49.407Z 21 Acked 1020 Write 5 Done Sent Done false
38952024-02-15T02:58:49.407Z 22 Acked 1021 Write 8 Done Sent Done false
38962024-02-15T02:58:49.407Z 23 Acked 1022 Flush 0 Done Sent Done false
38972024-02-15T02:58:49.407Z 24 Acked 1023 Write 4 Done Sent Done false
38982024-02-15T02:58:49.407Z 25 Acked 1024 Read 4 Done Sent Done false
38992024-02-15T02:58:49.407Z 26 Acked 1025 Flush 0 Done Sent Done false
39002024-02-15T02:58:49.407Z 27 Acked 1026 Write 6 Done Sent Done false
39012024-02-15T02:58:49.407Z 28 Acked 1027 Read 5 Done Sent Done false
39022024-02-15T02:58:49.407Z 29 Acked 1028 Read 2 Done Sent Done false
39032024-02-15T02:58:49.408Z 30 Acked 1029 Write 4 Sent Sent Sent false
39042024-02-15T02:58:49.408Z STATES DS:0 DS:1 DS:2 TOTAL
39052024-02-15T02:58:49.408Z New 0 0 0 0
39062024-02-15T02:58:49.408Z Sent 1 29 1 31
39072024-02-15T02:58:49.408Z Done 29 1 29 59
39082024-02-15T02:58:49.408Z Skipped 0 0 0 0
39092024-02-15T02:58:49.408Z Error 0 0 0 0
39102024-02-15T02:58:49.408ZLast Flush: 1025 0 1025
39112024-02-15T02:58:49.408ZDownstairs last five completed:
39122024-02-15T02:58:49.408ZUpstairs last five completed: 30 29 28 27 26
39132024-02-15T02:58:49.408Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:48.349696286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1271}
39142024-02-15T02:58:49.408ZWrote out file "/tmp/test_repair_verify.data"
39152024-02-15T02:58:49.412Z
39162024-02-15T02:58:49.415ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
39172024-02-15T02:58:49.429ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
39182024-02-15T02:58:49.429Z 0 000-019 14 14 14 25 25 25 F F F
39192024-02-15T02:58:49.429Z 1 020-039 27 27 27 53 53 53 F F F
39202024-02-15T02:58:49.430Z 2 040-059 22 22 22 40 40 40 T F T <---
39212024-02-15T02:58:49.430Z 3 060-079 31 31 31 59 59 59 F F F
39222024-02-15T02:58:49.430Z 4 080-099 31 31 31 59 59 59 F F F
39232024-02-15T02:58:49.430Z 5 100-119 32 32 32 62 62 62 F F F
39242024-02-15T02:58:49.433Z 6 120-139 32 32 32 62 62 62 F F F
39252024-02-15T02:58:49.433Z 7 140-159 32 32 32 62 62 62 F F F
39262024-02-15T02:58:49.433Z 8 160-179 34 32 34 66 62 66 F F F <---
39272024-02-15T02:58:49.433Z 9 180-199 30 30 30 57 57 57 F F F
39282024-02-15T02:58:49.433Z 10 200-219 24 24 24 45 45 45 F F F
39292024-02-15T02:58:49.434Z 11 220-239 34 26 34 66 51 66 F F F <---
39302024-02-15T02:58:49.434Z 12 240-259 34 32 34 66 62 66 F F F <---
39312024-02-15T02:58:49.434Z 13 260-279 32 32 32 62 62 62 F F F
39322024-02-15T02:58:49.434Z 14 280-299 34 30 34 66 57 66 F F F <---
39332024-02-15T02:58:49.434Z 15 300-319 32 32 32 62 62 62 F F F
39342024-02-15T02:58:49.434Z 16 320-339 26 26 26 52 52 52 F F F
39352024-02-15T02:58:49.434Z 17 340-359 33 33 33 63 63 63 F F F
39362024-02-15T02:58:49.434Z 18 360-379 28 28 28 55 55 55 F F F
39372024-02-15T02:58:49.434Z 19 380-399 34 32 34 65 61 65 F F F <---
39382024-02-15T02:58:49.434Z 20 400-419 34 24 34 66 45 66 F F F <---
39392024-02-15T02:58:49.434Z 21 420-439 29 29 29 56 56 56 F F F
39402024-02-15T02:58:49.434Z 22 440-459 32 32 32 62 62 62 F F F
39412024-02-15T02:58:49.434Z 23 460-479 34 33 34 64 63 64 T F T <---
39422024-02-15T02:58:49.434Z 24 480-499 29 29 29 56 56 56 F F F
39432024-02-15T02:58:49.434Z 25 500-519 34 32 34 66 60 66 F T F <---
39442024-02-15T02:58:49.434Z 26 520-539 32 32 32 62 62 62 F F F
39452024-02-15T02:58:49.435Z 27 540-559 31 31 31 59 59 59 F F F
39462024-02-15T02:58:49.435Z 28 560-579 33 33 33 63 63 63 F F F
39472024-02-15T02:58:49.435Z 29 580-599 34 26 34 67 50 67 F F F <---
39482024-02-15T02:58:49.435ZMax gen: 34, Max flush: 67
39492024-02-15T02:58:49.435ZError: Difference in extent metadata found!
39502024-02-15T02:58:49.435ZA Difference in extent metadata is expected here
39512024-02-15T02:58:49.435ZOn loop 17
39522024-02-15T02:58:49.435Z
39532024-02-15T02:58:49.438ZVerifying data now
39542024-02-15T02:58:54.962ZNo change after verify
39552024-02-15T02:58:54.962ZLoop: 17 Downstairs dump after verify (and repair):
39562024-02-15T02:58:54.979ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
39572024-02-15T02:58:54.979Z 0 000-019 14 14 14 25 25 25 F F F
39582024-02-15T02:58:54.979Z 1 020-039 27 27 27 53 53 53 F F F
39592024-02-15T02:58:54.979Z 2 040-059 35 35 35 68 68 68 F F F
39602024-02-15T02:58:54.979Z 3 060-079 31 31 31 59 59 59 F F F
39612024-02-15T02:58:54.979Z 4 080-099 31 31 31 59 59 59 F F F
39622024-02-15T02:58:54.979Z 5 100-119 32 32 32 62 62 62 F F F
39632024-02-15T02:58:54.979Z 6 120-139 32 32 32 62 62 62 F F F
39642024-02-15T02:58:54.979Z 7 140-159 32 32 32 62 62 62 F F F
39652024-02-15T02:58:54.979Z 8 160-179 34 34 34 66 66 66 F F F
39662024-02-15T02:58:54.979Z 9 180-199 30 30 30 57 57 57 F F F
39672024-02-15T02:58:54.979Z 10 200-219 24 24 24 45 45 45 F F F
39682024-02-15T02:58:54.980Z 11 220-239 34 34 34 66 66 66 F F F
39692024-02-15T02:58:54.980Z 12 240-259 34 34 34 66 66 66 F F F
39702024-02-15T02:58:54.980Z 13 260-279 32 32 32 62 62 62 F F F
39712024-02-15T02:58:54.980Z 14 280-299 34 34 34 66 66 66 F F F
39722024-02-15T02:58:54.980Z 15 300-319 32 32 32 62 62 62 F F F
39732024-02-15T02:58:54.984Z 16 320-339 26 26 26 52 52 52 F F F
39742024-02-15T02:58:54.984Z 17 340-359 33 33 33 63 63 63 F F F
39752024-02-15T02:58:54.984Z 18 360-379 28 28 28 55 55 55 F F F
39762024-02-15T02:58:54.984Z 19 380-399 34 34 34 65 65 65 F F F
39772024-02-15T02:58:54.984Z 20 400-419 34 34 34 66 66 66 F F F
39782024-02-15T02:58:54.984Z 21 420-439 29 29 29 56 56 56 F F F
39792024-02-15T02:58:54.984Z 22 440-459 32 32 32 62 62 62 F F F
39802024-02-15T02:58:54.984Z 23 460-479 35 35 35 68 68 68 F F F
39812024-02-15T02:58:54.984Z 24 480-499 29 29 29 56 56 56 F F F
39822024-02-15T02:58:54.984Z 25 500-519 34 34 34 66 66 66 F F F
39832024-02-15T02:58:54.984Z 26 520-539 32 32 32 62 62 62 F F F
39842024-02-15T02:58:54.984Z 27 540-559 31 31 31 59 59 59 F F F
39852024-02-15T02:58:54.985Z 28 560-579 33 33 33 63 63 63 F F F
39862024-02-15T02:58:54.985Z 29 580-599 34 34 34 67 67 67 F F F
39872024-02-15T02:58:54.985ZMax gen: 35, Max flush: 68
39882024-02-15T02:58:54.985Z
39892024-02-15T02:58:54.985ZBegin loop 18
39902024-02-15T02:58:54.985ZDownstairs to restart: 2
39912024-02-15T02:58:55.002Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.945644171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279}
39922024-02-15T02:58:55.002Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.945941954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279}
39932024-02-15T02:58:55.003Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.945961291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279}
39942024-02-15T02:58:55.122Z{"msg":"Crucible b91dc1c3-09b0-4f66-be98-06e3db399387 has session id: e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.945997327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279Crucible runtime is spawned
39952024-02-15T02:58:55.123Z,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
39962024-02-15T02:58:55.123Z{"msg":"Upstairs opts: Upstairs UUID: b91dc1c3-09b0-4f66-be98-06e3db399387, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946064938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
39972024-02-15T02:58:55.123Z{"msg":"Crucible stats registered with UUID: b91dc1c3-09b0-4f66-be98-06e3db399387","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946147358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
39982024-02-15T02:58:55.123Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946197861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279}
39992024-02-15T02:58:55.123Z{"msg":"b91dc1c3-09b0-4f66-be98-06e3db399387 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946339114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40002024-02-15T02:58:55.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946416284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40012024-02-15T02:58:55.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946442861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40022024-02-15T02:58:55.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946481206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40032024-02-15T02:58:55.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946507163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40042024-02-15T02:58:55.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.94652684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40052024-02-15T02:58:55.125Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946566315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40062024-02-15T02:58:55.125Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946596652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40072024-02-15T02:58:55.125Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946640326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40082024-02-15T02:58:55.125Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946669773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40092024-02-15T02:58:55.125Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946700539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40102024-02-15T02:58:55.125Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946725975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40112024-02-15T02:58:55.125Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.946785698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40122024-02-15T02:58:55.126Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.947561191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40132024-02-15T02:58:55.126Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.947602396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40142024-02-15T02:58:55.126Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.947623103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40152024-02-15T02:58:55.126Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.947661068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40162024-02-15T02:58:55.126Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.948250634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40172024-02-15T02:58:55.127Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.948425093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40182024-02-15T02:58:55.127Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.948826742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40192024-02-15T02:58:55.127Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.948908842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40202024-02-15T02:58:55.127Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.948951347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40212024-02-15T02:58:55.127Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.948968634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40222024-02-15T02:58:55.128Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.948991122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40232024-02-15T02:58:55.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.949022878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40242024-02-15T02:58:55.128Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.949044755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40252024-02-15T02:58:55.128Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.949058963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40262024-02-15T02:58:55.128Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.949554331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40272024-02-15T02:58:55.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950033721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40282024-02-15T02:58:55.129Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950077015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40292024-02-15T02:58:55.129Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 68, 59, 59, 62, 62, 62, 66, 57, 45, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950107692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40302024-02-15T02:58:55.129Z{"msg":"[0]R generation[0..12]: [14, 27, 35, 31, 31, 32, 32, 32, 34, 30, 24, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950132808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40312024-02-15T02:58:55.129Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950148077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40322024-02-15T02:58:55.129Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 68, 59, 59, 62, 62, 62, 66, 57, 45, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950169174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40332024-02-15T02:58:55.129Z{"msg":"[1]R generation[0..12]: [14, 27, 35, 31, 31, 32, 32, 32, 34, 30, 24, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950183062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40342024-02-15T02:58:55.130Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950197311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40352024-02-15T02:58:55.130Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 68, 59, 59, 62, 62, 62, 66, 57, 45, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950217938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
40362024-02-15T02:58:55.130Z,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40372024-02-15T02:58:55.130Z{"msg":"[2]R generation[0..12]: [14, 27, 35, 31, 31, 32, 32, 32, 34, 30, 24, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950235906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40382024-02-15T02:58:55.130Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950250294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40392024-02-15T02:58:55.130Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950270781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40402024-02-15T02:58:55.130Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.95028474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40412024-02-15T02:58:55.130Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950304297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40422024-02-15T02:58:55.131Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950317685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40432024-02-15T02:58:55.131Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950331704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40442024-02-15T02:58:55.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950343912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40452024-02-15T02:58:55.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.95036385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40462024-02-15T02:58:55.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950377338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40472024-02-15T02:58:55.132Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950390596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40482024-02-15T02:58:55.132Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950409544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40492024-02-15T02:58:55.132Z{"msg":"b91dc1c3-09b0-4f66-be98-06e3db399387 is now active with session: e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950422772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40502024-02-15T02:58:55.132Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.95044169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279,"session_id":"e17bbc55-9d7a-4ebf-b1ff-b7688b6c59ad"}
40512024-02-15T02:58:55.132Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:53.950454288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279}
40522024-02-15T02:58:55.132ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
40532024-02-15T02:58:55.132ZLoading write count information from file "/tmp/test_repair_verify.data"
40542024-02-15T02:58:55.132ZRun Repair workload
40552024-02-15T02:58:55.133Z01/30 Write block 284 len 16384 data: 4 4 4 4
40562024-02-15T02:58:55.141Z02/30 Write block 272 len 8192 data: 4 4
40572024-02-15T02:58:55.141Z03/30 Read block 323 len 36864
40582024-02-15T02:58:55.141Z04/30 Write block 190 len 24576 data: 4 4 5 7 6 6
40592024-02-15T02:58:55.142Z05/30 Write block 277 len 4096 data: 4
40602024-02-15T02:58:55.142Z06/30 Read block 167 len 8192
40612024-02-15T02:58:55.142Z07/30 Read block 253 len 32768
40622024-02-15T02:58:55.142Z08/30 Write block 105 len 4096 data: 5
40632024-02-15T02:58:55.142Z09/30 Write block 491 len 40960 data: 5 4 4 4 4 4 4 3 3 5
40642024-02-15T02:58:55.143Z10/30 Read block 219 len 28672
40652024-02-15T02:58:55.143Z11/30 Read block 334 len 12288
40662024-02-15T02:58:55.143Z12/30 Read block 438 len 16384
40672024-02-15T02:58:55.143Z13/30 Read block 578 len 24576
40682024-02-15T02:58:55.143Z14/30 Flush
40692024-02-15T02:58:55.143Z15/30 Write block 421 len 24576 data: 2 3 3 4 4 4
40702024-02-15T02:58:55.144Z16/30 Flush
40712024-02-15T02:58:55.144Z17/30 Write block 480 len 12288 data: 2 2 2
40722024-02-15T02:58:55.144Z18/30 Read block 165 len 28672
40732024-02-15T02:58:55.144Z19/30 Write block 527 len 12288 data: 5 5 5
40742024-02-15T02:58:55.144Z20/30 Write block 266 len 28672 data: 5 3 4 4 3 4 5
40752024-02-15T02:58:55.145Z21/30 Write block 377 len 24576 data: 4 5 4 4 4 4
40762024-02-15T02:58:55.145Z22/30 Write block 327 len 12288 data: 2 2 3
40772024-02-15T02:58:55.145Z23/30 Flush
40782024-02-15T02:58:55.145Z24/30 Flush
40792024-02-15T02:58:55.145Z25/30 Write block 537 len 12288 data: 5 4 4
40802024-02-15T02:58:55.145Z26/30 Write block 277 len 8192 data: 5 4
40812024-02-15T02:58:55.145Z27/30 Write block 323 len 40960 data: 4 4 4 4 3 3 4 4 5 5
40822024-02-15T02:58:55.145Z28/30 Read block 185 len 16384
40832024-02-15T02:58:55.146Z29/30 Write block 219 len 40960 data: 6 7 6 6 5 5 5 7 7 7
40842024-02-15T02:58:55.146Z30/30 Write block 460 len 24576 data: 3 3 4 3 3 4
40852024-02-15T02:58:55.146Z----------------------------------------------------------------
40862024-02-15T02:58:55.146Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
40872024-02-15T02:58:55.146ZStates: Active Active Active
40882024-02-15T02:58:55.146ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40892024-02-15T02:58:55.146Z 1 Acked 1000 Write 4 Done Done Sent false
40902024-02-15T02:58:55.147Z 2 Acked 1001 Write 2 Done Done Sent false
40912024-02-15T02:58:55.147Z 3 Acked 1002 Read 9 Done Done Sent false
40922024-02-15T02:58:55.147Z 4 Acked 1003 Write 6 Done Done Sent false
40932024-02-15T02:58:55.147Z 5 Acked 1004 Write 1 Done Done Sent false
40942024-02-15T02:58:55.147Z 6 Acked 1005 Read 2 Done Done Sent false
40952024-02-15T02:58:55.147Z 7 Acked 1006 Read 8 Done Done Sent false
40962024-02-15T02:58:55.147Z 8 Acked 1007 Write 1 Done Done Sent false
40972024-02-15T02:58:55.147Z 9 Acked 1008 Write 10 Done Done Sent false
40982024-02-15T02:58:55.147Z 10 Acked 1009 Read 7 Done Done Sent false
40992024-02-15T02:58:55.148Z 11 Acked 1010 Read 3 Done Done Sent false
41002024-02-15T02:58:55.148Z 12 Acked 1011 Read 4 Done Done Sent false
41012024-02-15T02:58:55.148Z 13 Acked 1012 Read 6 Done Done Sent false
41022024-02-15T02:58:55.148Z 14 Acked 1013 Flush 0 Done Done Sent false
41032024-02-15T02:58:55.149Z 15 Acked 1014 Write 6 Done Done Sent false
41042024-02-15T02:58:55.149Z 16 Acked 1015 Flush 0 Done Done Sent false
41052024-02-15T02:58:55.149Z 17 Acked 1016 Write 3 Done Done Sent false
41062024-02-15T02:58:55.149Z 18 Acked 1017 Read 7 Done Done Sent false
41072024-02-15T02:58:55.149Z 19 Acked 1018 Write 3 Done Done Sent false
41082024-02-15T02:58:55.150Z 20 Acked 1019 Write 7 Done Done Sent false
41092024-02-15T02:58:55.150Z 21 Acked 1020 Write 6 Done Done Sent false
41102024-02-15T02:58:55.150Z 22 Acked 1021 Write 3 Done Done Sent false
41112024-02-15T02:58:55.150Z 23 Acked 1022 Flush 0 Done Done Sent false
41122024-02-15T02:58:55.150Z 24 Acked 1023 Flush 0 Done Done Sent false
41132024-02-15T02:58:55.150Z 25 Acked 1024 Write 3 Done Done Sent false
41142024-02-15T02:58:55.151Z 26 Acked 1025 Write 2 Done Done Sent false
41152024-02-15T02:58:55.151Z 27 Acked 1026 Write 10 Done Done Sent false
41162024-02-15T02:58:55.151Z 28 Acked 1027 Read 4 Done Done Sent false
41172024-02-15T02:58:55.151Z 29 Acked 1028 Write 10 Sent Done Sent false
41182024-02-15T02:58:55.151Z 30 Acked 1029 Write 6 Sent Sent Sent false
41192024-02-15T02:58:55.151Z STATES DS:0 DS:1 DS:2 TOTAL
41202024-02-15T02:58:55.152Z New 0 0 0 0
41212024-02-15T02:58:55.152Z Sent 2 1 30 33
41222024-02-15T02:58:55.152Z Done 28 29 0 57
41232024-02-15T02:58:55.152Z Skipped 0 0 0 0
41242024-02-15T02:58:55.152Z Error 0 0 0 0
41252024-02-15T02:58:55.152ZLast Flush: 1023 1023 0
41262024-02-15T02:58:55.152ZDownstairs last five completed:
41272024-02-15T02:58:55.153ZUpstairs last five completed: 30 29 28 27 26
41282024-02-15T02:58:55.153Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:53.981351937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1279}
41292024-02-15T02:58:55.153ZWrote out file "/tmp/test_repair_verify.data"
41302024-02-15T02:58:55.153Z
41312024-02-15T02:58:55.153ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
41322024-02-15T02:58:55.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41332024-02-15T02:58:55.153Z 0 000-019 14 14 14 25 25 25 F F F
41342024-02-15T02:58:55.154Z 1 020-039 27 27 27 53 53 53 F F F
41352024-02-15T02:58:55.154Z 2 040-059 35 35 35 68 68 68 F F F
41362024-02-15T02:58:55.154Z 3 060-079 31 31 31 59 59 59 F F F
41372024-02-15T02:58:55.154Z 4 080-099 31 31 31 59 59 59 F F F
41382024-02-15T02:58:55.154Z 5 100-119 36 36 32 69 69 62 F F F <---
41392024-02-15T02:58:55.154Z 6 120-139 32 32 32 62 62 62 F F F
41402024-02-15T02:58:55.155Z 7 140-159 32 32 32 62 62 62 F F F
41412024-02-15T02:58:55.155Z 8 160-179 34 34 34 66 66 66 F F F
41422024-02-15T02:58:55.155Z 9 180-199 36 36 30 69 69 57 F F F <---
41432024-02-15T02:58:55.155Z 10 200-219 24 24 24 45 45 45 T T F <---
41442024-02-15T02:58:55.155Z 11 220-239 34 34 34 66 66 66 T T F <---
41452024-02-15T02:58:55.155Z 12 240-259 34 34 34 66 66 66 F F F
41462024-02-15T02:58:55.155Z 13 260-279 36 36 32 71 71 62 T T F <---
41472024-02-15T02:58:55.156Z 14 280-299 36 36 34 69 69 66 F F F <---
41482024-02-15T02:58:55.156Z 15 300-319 32 32 32 62 62 62 F F F
41492024-02-15T02:58:55.156Z 16 320-339 36 36 26 71 71 52 T T F <---
41502024-02-15T02:58:55.156Z 17 340-359 33 33 33 63 63 63 F F F
41512024-02-15T02:58:55.156Z 18 360-379 36 36 28 71 71 55 F F F <---
41522024-02-15T02:58:55.156Z 19 380-399 36 36 34 71 71 65 F F F <---
41532024-02-15T02:58:55.156Z 20 400-419 34 34 34 66 66 66 F F F
41542024-02-15T02:58:55.157Z 21 420-439 36 36 29 70 70 56 F F F <---
41552024-02-15T02:58:55.157Z 22 440-459 32 32 32 62 62 62 F F F
41562024-02-15T02:58:55.163Z 23 460-479 35 35 35 68 68 68 T T F <---
41572024-02-15T02:58:55.163Z 24 480-499 36 36 29 71 71 56 F F F <---
41582024-02-15T02:58:55.163Z 25 500-519 36 36 34 69 69 66 F F F <---
41592024-02-15T02:58:55.163Z 26 520-539 36 36 32 71 71 62 T T F <---
41602024-02-15T02:58:55.164Z 27 540-559 31 31 31 59 59 59 F F F
41612024-02-15T02:58:55.164Z 28 560-579 33 33 33 63 63 63 F F F
41622024-02-15T02:58:55.164Z 29 580-599 34 34 34 67 67 67 F F F
41632024-02-15T02:58:55.164ZMax gen: 36, Max flush: 71
41642024-02-15T02:58:55.164ZError: Difference in extent metadata found!
41652024-02-15T02:58:55.164ZA Difference in extent metadata is expected here
41662024-02-15T02:58:55.164ZOn loop 18
41672024-02-15T02:58:55.164Z
41682024-02-15T02:58:55.165ZVerifying data now
41692024-02-15T02:59:00.205ZNo change after verify
41702024-02-15T02:59:00.205ZLoop: 18 Downstairs dump after verify (and repair):
41712024-02-15T02:59:00.228ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41722024-02-15T02:59:00.228Z 0 000-019 14 14 14 25 25 25 F F F
41732024-02-15T02:59:00.228Z 1 020-039 27 27 27 53 53 53 F F F
41742024-02-15T02:59:00.228Z 2 040-059 35 35 35 68 68 68 F F F
41752024-02-15T02:59:00.228Z 3 060-079 31 31 31 59 59 59 F F F
41762024-02-15T02:59:00.228Z 4 080-099 31 31 31 59 59 59 F F F
41772024-02-15T02:59:00.228Z 5 100-119 36 36 36 69 69 69 F F F
41782024-02-15T02:59:00.229Z 6 120-139 32 32 32 62 62 62 F F F
41792024-02-15T02:59:00.229Z 7 140-159 32 32 32 62 62 62 F F F
41802024-02-15T02:59:00.229Z 8 160-179 34 34 34 66 66 66 F F F
41812024-02-15T02:59:00.229Z 9 180-199 36 36 36 69 69 69 F F F
41822024-02-15T02:59:00.229Z 10 200-219 37 37 37 72 72 72 F F F
41832024-02-15T02:59:00.229Z 11 220-239 37 37 37 72 72 72 F F F
41842024-02-15T02:59:00.229Z 12 240-259 34 34 34 66 66 66 F F F
41852024-02-15T02:59:00.229Z 13 260-279 37 37 37 72 72 72 F F F
41862024-02-15T02:59:00.229Z 14 280-299 36 36 36 69 69 69 F F F
41872024-02-15T02:59:00.229Z 15 300-319 32 32 32 62 62 62 F F F
41882024-02-15T02:59:00.229Z 16 320-339 37 37 37 72 72 72 F F F
41892024-02-15T02:59:00.229Z 17 340-359 33 33 33 63 63 63 F F F
41902024-02-15T02:59:00.230Z 18 360-379 36 36 36 71 71 71 F F F
41912024-02-15T02:59:00.230Z 19 380-399 36 36 36 71 71 71 F F F
41922024-02-15T02:59:00.230Z 20 400-419 34 34 34 66 66 66 F F F
41932024-02-15T02:59:00.230Z 21 420-439 36 36 36 70 70 70 F F F
41942024-02-15T02:59:00.230Z 22 440-459 32 32 32 62 62 62 F F F
41952024-02-15T02:59:00.230Z 23 460-479 37 37 37 72 72 72 F F F
41962024-02-15T02:59:00.230Z 24 480-499 36 36 36 71 71 71 F F F
41972024-02-15T02:59:00.230Z 25 500-519 36 36 36 69 69 69 F F F
41982024-02-15T02:59:00.230Z 26 520-539 37 37 37 72 72 72 F F F
41992024-02-15T02:59:00.230Z 27 540-559 31 31 31 59 59 59 F F F
42002024-02-15T02:59:00.231Z 28 560-579 33 33 33 63 63 63 F F F
42012024-02-15T02:59:00.231Z 29 580-599 34 34 34 67 67 67 F F F
42022024-02-15T02:59:00.231ZMax gen: 37, Max flush: 72
42032024-02-15T02:59:00.231Z
42042024-02-15T02:59:00.231ZBegin loop 19
42052024-02-15T02:59:00.231ZDownstairs to restart: 2
42062024-02-15T02:59:00.231Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.763718561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287}
42072024-02-15T02:59:00.231Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.7640406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287}
42082024-02-15T02:59:00.231Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764061618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287}
42092024-02-15T02:59:00.232Z{"msg":"Crucible 178a4e76-6c79-43df-8d2e-a4de1213b1db has session id: ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764080226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42102024-02-15T02:59:00.232Z{"msg":"Upstairs opts: Upstairs UUID: 178a4e76-6c79-43df-8d2e-a4de1213b1db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764130359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42112024-02-15T02:59:00.232Z{"msg":"Crucible stats registered with UUID: 178a4e76-6c79-43df-8d2e-a4de1213b1db","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764158706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42122024-02-15T02:59:00.232ZCrucible runtime is spawned
42132024-02-15T02:59:00.232Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764338763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287}
42142024-02-15T02:59:00.232Z{"msg":"178a4e76-6c79-43df-8d2e-a4de1213b1db active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764452449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42152024-02-15T02:59:00.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764479045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42162024-02-15T02:59:00.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764496343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42172024-02-15T02:59:00.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764511032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42182024-02-15T02:59:00.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.76452493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42192024-02-15T02:59:00.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764550027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42202024-02-15T02:59:00.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764575493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42212024-02-15T02:59:00.233Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.76460231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42222024-02-15T02:59:00.233Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764623058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42232024-02-15T02:59:00.233Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764637376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42242024-02-15T02:59:00.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.76484012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42252024-02-15T02:59:00.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.764903722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42262024-02-15T02:59:00.233Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:58.764962585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42272024-02-15T02:59:00.234Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:58.765012549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42282024-02-15T02:59:00.234Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:58.765049824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42292024-02-15T02:59:00.234Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.765070342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42302024-02-15T02:59:00.234Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.765106987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42312024-02-15T02:59:00.235Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.765128784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42322024-02-15T02:59:00.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.765651649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42332024-02-15T02:59:00.235Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.765688154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42342024-02-15T02:59:00.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.765703732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42352024-02-15T02:59:00.235Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.765768784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42362024-02-15T02:59:00.235Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.766303647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42372024-02-15T02:59:00.236Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.766344682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42382024-02-15T02:59:00.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.766922939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42392024-02-15T02:59:00.236Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.766963874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42402024-02-15T02:59:00.236Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.76699951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42412024-02-15T02:59:00.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.767020117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42422024-02-15T02:59:00.236Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.767034405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42432024-02-15T02:59:00.236Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:58.767055013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42442024-02-15T02:59:00.830Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.766791609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42452024-02-15T02:59:00.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.766977825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42462024-02-15T02:59:00.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.767835698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42472024-02-15T02:59:00.850Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.767871753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42482024-02-15T02:59:00.850Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.768352873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42492024-02-15T02:59:00.850Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.768915072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42502024-02-15T02:59:00.850Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.768965096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42512024-02-15T02:59:00.850Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 68, 59, 59, 69, 62, 62, 66, 69, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.76901634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42522024-02-15T02:59:00.850Z{"msg":"[0]R generation[0..12]: [14, 27, 35, 31, 31, 36, 32, 32, 34, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769038137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42532024-02-15T02:59:00.850Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769067804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42542024-02-15T02:59:00.850Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 68, 59, 59, 69, 62, 62, 66, 69, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769086051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42552024-02-15T02:59:00.851Z{"msg":"[1]R generation[0..12]: [14, 27, 35, 31, 31, 36, 32, 32, 34, 36, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769131316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42562024-02-15T02:59:00.851Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769150143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42572024-02-15T02:59:00.851Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 68, 59, 59, 69, 62, 62, 66, 69, 72, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.76917767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42582024-02-15T02:59:00.851Z{"msg":"[2]R generation[0..12]: [14, 27, 35, 31, 31, 36, 32, 32, 34, 36, 37, 37]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
42592024-02-15T02:59:00.851Z,"time":"2024-02-15T02:58:59.769195468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42602024-02-15T02:59:00.851Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769273818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42612024-02-15T02:59:00.851Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769293815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42622024-02-15T02:59:00.851Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769311333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42632024-02-15T02:59:00.851Z{"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769338559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42642024-02-15T02:59:00.851Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769355497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42652024-02-15T02:59:00.851Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769381354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42662024-02-15T02:59:00.851Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769396932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42672024-02-15T02:59:00.851Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.76941373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42682024-02-15T02:59:00.851Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769438467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42692024-02-15T02:59:00.851Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769455965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42702024-02-15T02:59:00.851Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769472103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42712024-02-15T02:59:00.852Z{"msg":"178a4e76-6c79-43df-8d2e-a4de1213b1db is now active with session: ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.76949673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42722024-02-15T02:59:00.852Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769512678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287,"session_id":"ebec7575-bc64-4abd-9c66-c3c5bc5e5eb0"}
42732024-02-15T02:59:00.852Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:58:59.769536165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287}
42742024-02-15T02:59:00.852ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
42752024-02-15T02:59:00.852ZLoading write count information from file "/tmp/test_repair_verify.data"
42762024-02-15T02:59:00.852ZRun Repair workload
42772024-02-15T02:59:00.852Z01/30 Write block 74 len 28672 data: 2 3 4 4 3 3 3
42782024-02-15T02:59:00.852Z02/30 Read block 513 len 16384
42792024-02-15T02:59:00.852Z03/30 Write block 140 len 32768 data: 4 4 3 3 2 3 3 5
42802024-02-15T02:59:00.852Z04/30 Flush
42812024-02-15T02:59:00.852Z05/30 Write block 484 len 32768 data: 2 2 3 5 4 5 6 6
42822024-02-15T02:59:00.852Z06/30 Read block 111 len 12288
42832024-02-15T02:59:00.852Z07/30 Read block 587 len 4096
42842024-02-15T02:59:00.852Z08/30 Read block 100 len 32768
42852024-02-15T02:59:00.852Z09/30 Read block 204 len 40960
42862024-02-15T02:59:00.852Z10/30 Read block 300 len 24576
42872024-02-15T02:59:00.852Z11/30 Read block 550 len 24576
42882024-02-15T02:59:00.853Z12/30 Write block 362 len 12288 data: 4 3 3
42892024-02-15T02:59:00.894Z13/30 Read block 454 len 8192
42902024-02-15T02:59:00.894Z14/30 Write block 395 len 24576 data: 3 3 3 2 3 2
42912024-02-15T02:59:00.894Z15/30 Write block 389 len 28672 data: 4 4 4 5 4 3 4
42922024-02-15T02:59:00.894Z16/30 Read block 34 len 4096
42932024-02-15T02:59:00.895Z17/30 Read block 179 len 28672
42942024-02-15T02:59:00.895Z18/30 Read block 474 len 40960
42952024-02-15T02:59:00.895Z19/30 Read block 215 len 36864
42962024-02-15T02:59:00.895Z20/30 Read block 529 len 8192
42972024-02-15T02:59:00.895Z21/30 Read block 393 len 32768
42982024-02-15T02:59:00.895Z22/30 Read block 160 len 40960
42992024-02-15T02:59:00.895Z23/30 Write block 231 len 36864 data: 4 5 5 5 5 6 4 2 2
43002024-02-15T02:59:00.895Z24/30 Write block 213 len 40960 data: 3 4 3 3 4 4 7 8 7 7
43012024-02-15T02:59:00.895Z25/30 Read block 481 len 32768
43022024-02-15T02:59:00.895Z26/30 Read block 348 len 32768
43032024-02-15T02:59:00.895Z27/30 Read block 81 len 8192
43042024-02-15T02:59:00.895Z28/30 Read block 487 len 16384
43052024-02-15T02:59:00.895Z29/30 Write block 313 len 16384 data: 6 4 3 3
43062024-02-15T02:59:00.896Z30/30 Read block 360 len 28672
43072024-02-15T02:59:00.896Z----------------------------------------------------------------
43082024-02-15T02:59:00.896Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
43092024-02-15T02:59:00.896ZStates: Active Active Active
43102024-02-15T02:59:00.896ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
43112024-02-15T02:59:00.896Z 1 Acked 1000 Write 7 Done Done Done false
43122024-02-15T02:59:00.896Z 2 Acked 1001 Read 4 Done Done Done false
43132024-02-15T02:59:00.896Z 3 Acked 1002 Write 8 Done Done Sent false
43142024-02-15T02:59:00.896Z 4 Acked 1003 Flush 0 Done Done Sent false
43152024-02-15T02:59:00.896Z 5 Acked 1004 Write 8 Done Done Sent false
43162024-02-15T02:59:00.896Z 6 Acked 1005 Read 3 Done Done Sent false
43172024-02-15T02:59:00.897Z 7 Acked 1006 Read 1 Done Done Sent false
43182024-02-15T02:59:00.897Z 8 Acked 1007 Read 8 Done Done Sent false
43192024-02-15T02:59:00.897Z 9 Acked 1008 Read 10 Done Done Sent false
43202024-02-15T02:59:00.897Z 10 Acked 1009 Read 6 Done Done Sent false
43212024-02-15T02:59:00.897Z 11 Acked 1010 Read 6 Done Done Sent false
43222024-02-15T02:59:00.897Z 12 Acked 1011 Write 3 Done Done Sent false
43232024-02-15T02:59:00.897Z 13 Acked 1012 Read 2 Done Done Sent false
43242024-02-15T02:59:00.897Z 14 Acked 1013 Write 6 Done Done Sent false
43252024-02-15T02:59:00.897Z 15 Acked 1014 Write 7 Done Done Sent false
43262024-02-15T02:59:00.897Z 16 Acked 1015 Read 1 Done Done Sent false
43272024-02-15T02:59:00.897Z 17 Acked 1016 Read 7 Done Done Sent false
43282024-02-15T02:59:00.897Z 18 Acked 1017 Read 10 Done Done Sent false
43292024-02-15T02:59:00.897Z 19 Acked 1018 Read 9 Done Done Sent false
43302024-02-15T02:59:00.897Z 20 Acked 1019 Read 2 Done Done Sent false
43312024-02-15T02:59:00.898Z 21 Acked 1020 Read 8 Done Done Sent false
43322024-02-15T02:59:00.898Z 22 Acked 1021 Read 10 Done Done Sent false
43332024-02-15T02:59:00.898Z 23 Acked 1022 Write 9 Done Done Sent false
43342024-02-15T02:59:00.898Z 24 Acked 1023 Write 10 Done Done Sent false
43352024-02-15T02:59:00.898Z 25 Acked 1024 Read 8 Done Done Sent false
43362024-02-15T02:59:00.898Z 26 Acked 1025 Read 8 Done Done Sent false
43372024-02-15T02:59:00.898Z 27 Acked 1026 Read 2 Done Done Sent false
43382024-02-15T02:59:00.898Z 28 Acked 1027 Read 4 Done Done Sent false
43392024-02-15T02:59:00.901Z 29 Acked 1028 Write 4 Done Done Sent false
43402024-02-15T02:59:00.901Z 30 Acked 1029 Read 7 Done Sent Sent false
43412024-02-15T02:59:00.902Z STATES DS:0 DS:1 DS:2 TOTAL
43422024-02-15T02:59:00.902Z New 0 0 0 0
43432024-02-15T02:59:00.902Z Sent 0 1 28 29
43442024-02-15T02:59:00.902Z Done 30 29 2 61
43452024-02-15T02:59:00.902Z Skipped 0 0 0 0
43462024-02-15T02:59:00.902Z Error 0 0 0 0
43472024-02-15T02:59:00.902ZLast Flush: 1003 1003 0
43482024-02-15T02:59:00.903ZDownstairs last five completed:
43492024-02-15T02:59:00.903ZUpstairs last five completed: 30 29 28 27 26
43502024-02-15T02:59:00.903Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:58:59.800586274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1287}
43512024-02-15T02:59:00.903ZWrote out file "/tmp/test_repair_verify.data"
43522024-02-15T02:59:00.904Z
43532024-02-15T02:59:00.963ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
43542024-02-15T02:59:00.986ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43552024-02-15T02:59:00.987Z 0 000-019 14 14 14 25 25 25 F F F
43562024-02-15T02:59:00.987Z 1 020-039 27 27 27 53 53 53 F F F
43572024-02-15T02:59:00.987Z 2 040-059 35 35 35 68 68 68 F F F
43582024-02-15T02:59:00.987Z 3 060-079 38 38 31 73 73 59 F F T <---
43592024-02-15T02:59:00.987Z 4 080-099 38 38 31 73 73 59 F F T <---
43602024-02-15T02:59:00.987Z 5 100-119 36 36 36 69 69 69 F F F
43612024-02-15T02:59:00.987Z 6 120-139 32 32 32 62 62 62 F F F
43622024-02-15T02:59:00.987Z 7 140-159 38 38 32 73 73 62 F F F <---
43632024-02-15T02:59:00.988Z 8 160-179 34 34 34 66 66 66 F F F
43642024-02-15T02:59:00.988Z 9 180-199 36 36 36 69 69 69 F F F
43652024-02-15T02:59:00.988Z 10 200-219 37 37 37 72 72 72 T T F <---
43662024-02-15T02:59:00.988Z 11 220-239 37 37 37 72 72 72 T T F <---
43672024-02-15T02:59:00.988Z 12 240-259 34 34 34 66 66 66 F F F
43682024-02-15T02:59:00.988Z 13 260-279 37 37 37 72 72 72 F F F
43692024-02-15T02:59:00.988Z 14 280-299 36 36 36 69 69 69 F F F
43702024-02-15T02:59:00.988Z 15 300-319 32 32 32 62 62 62 T T F <---
43712024-02-15T02:59:00.988Z 16 320-339 37 37 37 72 72 72 F F F
43722024-02-15T02:59:00.988Z 17 340-359 33 33 33 63 63 63 F F F
43732024-02-15T02:59:00.988Z 18 360-379 36 36 36 71 71 71 T T F <---
43742024-02-15T02:59:00.988Z 19 380-399 36 36 36 71 71 71 T T F <---
43752024-02-15T02:59:00.988Z 20 400-419 34 34 34 66 66 66 T T F <---
43762024-02-15T02:59:00.989Z 21 420-439 36 36 36 70 70 70 F F F
43772024-02-15T02:59:00.989Z 22 440-459 32 32 32 62 62 62 F F F
43782024-02-15T02:59:00.989Z 23 460-479 37 37 37 72 72 72 F F F
43792024-02-15T02:59:00.989Z 24 480-499 36 36 36 71 71 71 T T F <---
43802024-02-15T02:59:00.989Z 25 500-519 36 36 36 69 69 69 F F F
43812024-02-15T02:59:00.989Z 26 520-539 37 37 37 72 72 72 F F F
43822024-02-15T02:59:00.989Z 27 540-559 31 31 31 59 59 59 F F F
43832024-02-15T02:59:00.989Z 28 560-579 33 33 33 63 63 63 F F F
43842024-02-15T02:59:00.989Z 29 580-599 34 34 34 67 67 67 F F F
43852024-02-15T02:59:00.990ZMax gen: 38, Max flush: 73
43862024-02-15T02:59:00.990ZError: Difference in extent metadata found!
43872024-02-15T02:59:00.990ZA Difference in extent metadata is expected here
43882024-02-15T02:59:00.990ZOn loop 19
43892024-02-15T02:59:00.990Z
43902024-02-15T02:59:00.990ZVerifying data now
43912024-02-15T02:59:06.494ZNo change after verify
43922024-02-15T02:59:06.494ZLoop: 19 Downstairs dump after verify (and repair):
43932024-02-15T02:59:06.514ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43942024-02-15T02:59:06.514Z 0 000-019 14 14 14 25 25 25 F F F
43952024-02-15T02:59:06.515Z 1 020-039 27 27 27 53 53 53 F F F
43962024-02-15T02:59:06.515Z 2 040-059 35 35 35 68 68 68 F F F
43972024-02-15T02:59:06.515Z 3 060-079 38 38 38 73 73 73 F F F
43982024-02-15T02:59:06.516Z 4 080-099 38 38 38 73 73 73 F F F
43992024-02-15T02:59:06.516Z 5 100-119 36 36 36 69 69 69 F F F
44002024-02-15T02:59:06.516Z 6 120-139 32 32 32 62 62 62 F F F
44012024-02-15T02:59:06.516Z 7 140-159 38 38 38 73 73 73 F F F
44022024-02-15T02:59:06.517Z 8 160-179 34 34 34 66 66 66 F F F
44032024-02-15T02:59:06.517Z 9 180-199 36 36 36 69 69 69 F F F
44042024-02-15T02:59:06.517Z 10 200-219 39 39 39 74 74 74 F F F
44052024-02-15T02:59:06.518Z 11 220-239 39 39 39 74 74 74 F F F
44062024-02-15T02:59:06.518Z 12 240-259 34 34 34 66 66 66 F F F
44072024-02-15T02:59:06.518Z 13 260-279 37 37 37 72 72 72 F F F
44082024-02-15T02:59:06.518Z 14 280-299 36 36 36 69 69 69 F F F
44092024-02-15T02:59:06.518Z 15 300-319 39 39 39 74 74 74 F F F
44102024-02-15T02:59:06.518Z 16 320-339 37 37 37 72 72 72 F F F
44112024-02-15T02:59:06.518Z 17 340-359 33 33 33 63 63 63 F F F
44122024-02-15T02:59:06.519Z 18 360-379 39 39 39 74 74 74 F F F
44132024-02-15T02:59:06.519Z 19 380-399 39 39 39 74 74 74 F F F
44142024-02-15T02:59:06.519Z 20 400-419 39 39 39 74 74 74 F F F
44152024-02-15T02:59:06.519Z 21 420-439 36 36 36 70 70 70 F F F
44162024-02-15T02:59:06.519Z 22 440-459 32 32 32 62 62 62 F F F
44172024-02-15T02:59:06.519Z 23 460-479 37 37 37 72 72 72 F F F
44182024-02-15T02:59:06.519Z 24 480-499 39 39 39 74 74 74 F F F
44192024-02-15T02:59:06.519Z 25 500-519 36 36 36 69 69 69 F F F
44202024-02-15T02:59:06.519Z 26 520-539 37 37 37 72 72 72 F F F
44212024-02-15T02:59:06.519Z 27 540-559 31 31 31 59 59 59 F F F
44222024-02-15T02:59:06.519Z 28 560-579 33 33 33 63 63 63 F F F
44232024-02-15T02:59:06.519Z 29 580-599 34 34 34 67 67 67 F F F
44242024-02-15T02:59:06.520ZMax gen: 39, Max flush: 74
44252024-02-15T02:59:06.520Z
44262024-02-15T02:59:06.520ZBegin loop 20
44272024-02-15T02:59:06.520ZDownstairs to restart: 1
44282024-02-15T02:59:06.520Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.420850992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295}
44292024-02-15T02:59:06.520Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421288248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295}
44302024-02-15T02:59:06.520ZCrucible runtime is spawned
44312024-02-15T02:59:06.520Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421335802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295}
44322024-02-15T02:59:06.521Z{"msg":"Crucible fa7abd78-0a7d-4acb-a21d-676f0a765b26 has session id: 10d7b0c8-778b-4ba6-bf08-41ecc262d131","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421378296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44332024-02-15T02:59:06.521Z{"msg":"Upstairs opts: Upstairs UUID: fa7abd78-0a7d-4acb-a21d-676f0a765b26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.42142939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44342024-02-15T02:59:06.521Z{"msg":"Crucible stats registered with UUID: fa7abd78-0a7d-4acb-a21d-676f0a765b26","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421464416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44352024-02-15T02:59:06.521Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421489342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295}
44362024-02-15T02:59:06.521Z{"msg":"fa7abd78-0a7d-4acb-a21d-676f0a765b26 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421552534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44372024-02-15T02:59:06.521Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.42158952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44382024-02-15T02:59:06.521Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421634984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44392024-02-15T02:59:06.521Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.42166708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44402024-02-15T02:59:06.521Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421700786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44412024-02-15T02:59:06.521Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421751289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44422024-02-15T02:59:06.521Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421865575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44432024-02-15T02:59:06.522Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421902771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44442024-02-15T02:59:06.522Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.421968672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44452024-02-15T02:59:06.522Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422059401Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44462024-02-15T02:59:06.522Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422095326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44472024-02-15T02:59:06.522Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422122533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44482024-02-15T02:59:06.522Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:05.422154469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44492024-02-15T02:59:06.522Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:05.422199953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44502024-02-15T02:59:06.522Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:05.422276634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44512024-02-15T02:59:06.522Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422353624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44522024-02-15T02:59:06.522Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422404408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44532024-02-15T02:59:06.523Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422453292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44542024-02-15T02:59:06.523Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422672924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44552024-02-15T02:59:06.523Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422711319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44562024-02-15T02:59:06.523Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422727657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44572024-02-15T02:59:06.523Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.422758713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44582024-02-15T02:59:06.523Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.423337661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44592024-02-15T02:59:06.523Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.423386825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44602024-02-15T02:59:06.523Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.42390502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44612024-02-15T02:59:06.524Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.423941315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44622024-02-15T02:59:06.524Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.423956613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44632024-02-15T02:59:06.524Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.424048972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44642024-02-15T02:59:06.524Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.424086877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44652024-02-15T02:59:06.524Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:05.424104145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44662024-02-15T02:59:07.519Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.423817374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44672024-02-15T02:59:07.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.423910072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44682024-02-15T02:59:07.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.42472133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44692024-02-15T02:59:07.527Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.424759366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44702024-02-15T02:59:07.527Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.42528582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44712024-02-15T02:59:07.527Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.42584344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44722024-02-15T02:59:07.527Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.425877126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44732024-02-15T02:59:07.527Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 68, 73, 73, 69, 62, 73, 66, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.425897693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44742024-02-15T02:59:07.527Z{"msg":"[0]R generation[0..12]: [14, 27, 35, 38, 38, 36, 32, 38, 34, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.425928249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44752024-02-15T02:59:07.527Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.425943817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44762024-02-15T02:59:07.527Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 68, 73, 73, 69, 62, 73, 66, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.425958345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44772024-02-15T02:59:07.527Z{"msg":"[1]R generation[0..12]: [14, 27, 35, 38, 38, 36, 32, 38, 34, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.425979673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44782024-02-15T02:59:07.527Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.425995531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44792024-02-15T02:59:07.528Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 68, 73, 73, 69, 62, 73, 66, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426018938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44802024-02-15T02:59:07.528Z{"msg":"[2]R generation[0..12]: [14, 27, 35, 38, 38, 36, 32, 38, 34, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426041555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44812024-02-15T02:59:07.528Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426056303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44822024-02-15T02:59:07.528Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426069831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44832024-02-15T02:59:07.528Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426090909Z"Wait for a query_work_queue command to finish before sending IO
44842024-02-15T02:59:07.528Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44852024-02-15T02:59:07.528Z{"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426108917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44862024-02-15T02:59:07.528Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426122245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44872024-02-15T02:59:07.528Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426135613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44882024-02-15T02:59:07.528Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426148052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44892024-02-15T02:59:07.528Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426168639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44902024-02-15T02:59:07.528Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426182287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44912024-02-15T02:59:07.529Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426196086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44922024-02-15T02:59:07.529Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426215113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44932024-02-15T02:59:07.529Z{"msg":"fa7abd78-0a7d-4acb-a21d-676f0a765b26 is now active with session: 10d7b0c8-778b-4ba6-bf08-41ecc262d131","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426228591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44942024-02-15T02:59:07.529Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426249759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295,"session_id":"10d7b0c8-778b-4ba6-bf08-41ecc262d131"}
44952024-02-15T02:59:07.529Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:06.426279305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295}
44962024-02-15T02:59:07.529ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
44972024-02-15T02:59:07.529ZLoading write count information from file "/tmp/test_repair_verify.data"
44982024-02-15T02:59:07.529ZRun Repair workload
44992024-02-15T02:59:07.529Z01/30 Write block 441 len 36864 data: 4 5 6 5 7 7 7 6 4
45002024-02-15T02:59:07.529Z02/30 Flush
45012024-02-15T02:59:07.529Z03/30 Write block 405 len 32768 data: 3 4 5 5 6 6 6 7
45022024-02-15T02:59:07.529Z04/30 Write block 509 len 24576 data: 3 3 4 5 5 5
45032024-02-15T02:59:07.530Z05/30 Flush
45042024-02-15T02:59:07.530Z06/30 Write block 298 len 40960 data: 4 5 5 7 7 8 9 10 8 7
45052024-02-15T02:59:07.530Z07/30 Read block 299 len 16384
45062024-02-15T02:59:07.530Z08/30 Write block 546 len 12288 data: 2 2 3
45072024-02-15T02:59:07.530Z09/30 Write block 494 len 4096 data: 5
45082024-02-15T02:59:07.531Z10/30 Write block 71 len 8192 data: 3 2
45092024-02-15T02:59:07.531Z11/30 Read block 468 len 12288
45102024-02-15T02:59:07.532Z12/30 Write block 321 len 32768 data: 3 3 5 5 5 5 4 4
45112024-02-15T02:59:07.532Z13/30 Read block 80 len 20480
45122024-02-15T02:59:07.532Z14/30 Write block 140 len 40960 data: 5 5 4 4 3 4 4 6 5 4
45132024-02-15T02:59:07.532Z15/30 Read block 168 len 32768
45142024-02-15T02:59:07.532Z16/30 Read block 48 len 40960
45152024-02-15T02:59:07.532Z17/30 Read block 393 len 40960
45162024-02-15T02:59:07.532Z18/30 Read block 88 len 24576
45172024-02-15T02:59:07.532Z19/30 Write block 366 len 36864 data: 4 4 3 3 3 3 3 3 3
45182024-02-15T02:59:07.532Z20/30 Read block 420 len 28672
45192024-02-15T02:59:07.533Z21/30 Read block 161 len 32768
45202024-02-15T02:59:07.533Z22/30 Flush
45212024-02-15T02:59:07.533Z23/30 Write block 249 len 36864 data: 5 4 3 4 4 5 4 4 5
45222024-02-15T02:59:07.533Z24/30 Write block 557 len 16384 data: 5 5 6 5
45232024-02-15T02:59:07.533Z25/30 Write block 398 len 36864 data: 3 4 3 2 2 2 2 4 5
45242024-02-15T02:59:07.533Z26/30 Read block 15 len 12288
45252024-02-15T02:59:07.533Z27/30 Write block 564 len 8192 data: 5 5
45262024-02-15T02:59:07.533Z28/30 Read block 191 len 12288
45272024-02-15T02:59:07.534Z29/30 Write block 44 len 20480 data: 4 5 5 5 4
45282024-02-15T02:59:07.534Z30/30 Write block 573 len 32768 data: 5 5 6 6 5 5 3 3
45292024-02-15T02:59:07.534Z----------------------------------------------------------------
45302024-02-15T02:59:07.534Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:28
45312024-02-15T02:59:07.534ZStates: Active Active Active
45322024-02-15T02:59:07.534ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
45332024-02-15T02:59:07.534Z 3 Acked 1002 Write 8 Done Sent Done false
45342024-02-15T02:59:07.534Z 4 Acked 1003 Write 6 Done Sent Done false
45352024-02-15T02:59:07.534Z 5 Acked 1004 Flush 0 Done Sent Done false
45362024-02-15T02:59:07.534Z 6 Acked 1005 Write 10 Done Sent Done false
45372024-02-15T02:59:07.534Z 7 Acked 1006 Read 4 Done Sent Done false
45382024-02-15T02:59:07.534Z 8 Acked 1007 Write 3 Done Sent Done false
45392024-02-15T02:59:07.534Z 9 Acked 1008 Write 1 Done Sent Done false
45402024-02-15T02:59:07.535Z 10 Acked 1009 Write 2 Done Sent Done false
45412024-02-15T02:59:07.535Z 11 Acked 1010 Read 3 Done Sent Done false
45422024-02-15T02:59:07.535Z 12 Acked 1011 Write 8 Done Sent Done false
45432024-02-15T02:59:07.535Z 13 Acked 1012 Read 5 Done Sent Done false
45442024-02-15T02:59:07.536Z 14 Acked 1013 Write 10 Done Sent Done false
45452024-02-15T02:59:07.536Z 15 Acked 1014 Read 8 Done Sent Done false
45462024-02-15T02:59:07.536Z 16 Acked 1015 Read 10 Done Sent Done false
45472024-02-15T02:59:07.536Z 17 Acked 1016 Read 10 Done Sent Done false
45482024-02-15T02:59:07.536Z 18 Acked 1017 Read 6 Done Sent Done false
45492024-02-15T02:59:07.536Z 19 Acked 1018 Write 9 Done Sent Done false
45502024-02-15T02:59:07.536Z 20 Acked 1019 Read 7 Done Sent Done false
45512024-02-15T02:59:07.536Z 21 Acked 1020 Read 8 Done Sent Done false
45522024-02-15T02:59:07.536Z 22 Acked 1021 Flush 0 Done Sent Done false
45532024-02-15T02:59:07.536Z 23 Acked 1022 Write 9 Done Sent Done false
45542024-02-15T02:59:07.536Z 24 Acked 1023 Write 4 Done Sent Done false
45552024-02-15T02:59:07.537Z 25 Acked 1024 Write 9 Done Sent Done false
45562024-02-15T02:59:07.537Z 26 Acked 1025 Read 3 Done Sent Done false
45572024-02-15T02:59:07.537Z 27 Acked 1026 Write 2 Done Sent Done false
45582024-02-15T02:59:07.537Z 28 Acked 1027 Read 3 Done Sent Done false
45592024-02-15T02:59:07.537Z 29 Acked 1028 Write 5 Done Sent Done false
45602024-02-15T02:59:07.537Z 30 Acked 1029 Write 8 Sent Sent Sent false
45612024-02-15T02:59:07.537Z STATES DS:0 DS:1 DS:2 TOTAL
45622024-02-15T02:59:07.537Z New 0 0 0 0
45632024-02-15T02:59:07.537Z Sent 1 28 1 30
45642024-02-15T02:59:07.537Z Done 27 0 27 54
45652024-02-15T02:59:07.537Z Skipped 0 0 0 0
45662024-02-15T02:59:07.537Z Error 0 0 0 0
45672024-02-15T02:59:07.537ZLast Flush: 1021 1001 1021
45682024-02-15T02:59:07.571ZDownstairs last five completed: 1001 1000
45692024-02-15T02:59:07.571ZUpstairs last five completed: 30 29 28 27 26
45702024-02-15T02:59:07.571Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:06.460989426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1295}
45712024-02-15T02:59:07.571ZWrote out file "/tmp/test_repair_verify.data"
45722024-02-15T02:59:07.571Z
45732024-02-15T02:59:07.571ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
45742024-02-15T02:59:07.571ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45752024-02-15T02:59:07.571Z 0 000-019 14 14 14 25 25 25 F F F
45762024-02-15T02:59:07.571Z 1 020-039 27 27 27 53 53 53 F F F
45772024-02-15T02:59:07.571Z 2 040-059 35 35 35 68 68 68 T F T <---
45782024-02-15T02:59:07.571Z 3 060-079 40 38 40 77 73 77 F F F <---
45792024-02-15T02:59:07.571Z 4 080-099 38 38 38 73 73 73 F F F
45802024-02-15T02:59:07.571Z 5 100-119 36 36 36 69 69 69 F F F
45812024-02-15T02:59:07.572Z 6 120-139 32 32 32 62 62 62 F F F
45822024-02-15T02:59:07.572Z 7 140-159 40 38 40 77 73 77 F F F <---
45832024-02-15T02:59:07.572Z 8 160-179 34 34 34 66 66 66 F F F
45842024-02-15T02:59:07.572Z 9 180-199 36 36 36 69 69 69 F F F
45852024-02-15T02:59:07.572Z 10 200-219 39 39 39 74 74 74 F F F
45862024-02-15T02:59:07.572Z 11 220-239 39 39 39 74 74 74 F F F
45872024-02-15T02:59:07.572Z 12 240-259 34 34 34 66 66 66 T F T <---
45882024-02-15T02:59:07.572Z 13 260-279 37 37 37 72 72 72 F F F
45892024-02-15T02:59:07.572Z 14 280-299 40 36 40 77 69 77 F F F <---
45902024-02-15T02:59:07.572Z 15 300-319 40 39 40 77 74 77 F F F <---
45912024-02-15T02:59:07.572Z 16 320-339 40 37 40 77 72 77 F F F <---
45922024-02-15T02:59:07.572Z 17 340-359 33 33 33 63 63 63 F F F
45932024-02-15T02:59:07.572Z 18 360-379 40 39 40 77 74 77 F F F <---
45942024-02-15T02:59:07.573Z 19 380-399 39 39 39 74 74 74 T F T <---
45952024-02-15T02:59:07.573Z 20 400-419 40 39 40 76 74 76 T F T <---
45962024-02-15T02:59:07.573Z 21 420-439 36 36 36 70 70 70 F F F
45972024-02-15T02:59:07.573Z 22 440-459 40 40 40 75 75 75 F F F
45982024-02-15T02:59:07.573Z 23 460-479 37 37 37 72 72 72 F F F
45992024-02-15T02:59:07.573Z 24 480-499 40 39 40 77 74 77 F F F <---
46002024-02-15T02:59:07.573Z 25 500-519 40 36 40 76 69 76 F F F <---
46012024-02-15T02:59:07.573Z 26 520-539 37 37 37 72 72 72 F F F
46022024-02-15T02:59:07.573Z 27 540-559 40 31 40 77 59 77 T F T <---
46032024-02-15T02:59:07.573Z 28 560-579 33 33 33 63 63 63 T F T <---
46042024-02-15T02:59:07.573Z 29 580-599 34 34 34 67 67 67 T F T <---
46052024-02-15T02:59:07.573ZMax gen: 40, Max flush: 77
46062024-02-15T02:59:07.573ZError: Difference in extent metadata found!
46072024-02-15T02:59:07.573ZA Difference in extent metadata is expected here
46082024-02-15T02:59:07.574ZOn loop 20
46092024-02-15T02:59:07.574Z
46102024-02-15T02:59:07.574ZVerifying data now
46112024-02-15T02:59:13.083ZNo change after verify
46122024-02-15T02:59:13.083ZLoop: 20 Downstairs dump after verify (and repair):
46132024-02-15T02:59:13.083ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46142024-02-15T02:59:13.083Z 0 000-019 14 14 14 25 25 25 F F F
46152024-02-15T02:59:13.083Z 1 020-039 27 27 27 53 53 53 F F F
46162024-02-15T02:59:13.083Z 2 040-059 41 41 41 78 78 78 F F F
46172024-02-15T02:59:13.083Z 3 060-079 40 40 40 77 77 77 F F F
46182024-02-15T02:59:13.083Z 4 080-099 38 38 38 73 73 73 F F F
46192024-02-15T02:59:13.084Z 5 100-119 36 36 36 69 69 69 F F F
46202024-02-15T02:59:13.084Z 6 120-139 32 32 32 62 62 62 F F F
46212024-02-15T02:59:13.084Z 7 140-159 40 40 40 77 77 77 F F F
46222024-02-15T02:59:13.084Z 8 160-179 34 34 34 66 66 66 F F F
46232024-02-15T02:59:13.084Z 9 180-199 36 36 36 69 69 69 F F F
46242024-02-15T02:59:13.084Z 10 200-219 39 39 39 74 74 74 F F F
46252024-02-15T02:59:13.084Z 11 220-239 39 39 39 74 74 74 F F F
46262024-02-15T02:59:13.084Z 12 240-259 41 41 41 78 78 78 F F F
46272024-02-15T02:59:13.085Z 13 260-279 37 37 37 72 72 72 F F F
46282024-02-15T02:59:13.085Z 14 280-299 40 40 40 77 77 77 F F F
46292024-02-15T02:59:13.085Z 15 300-319 40 40 40 77 77 77 F F F
46302024-02-15T02:59:13.085Z 16 320-339 40 40 40 77 77 77 F F F
46312024-02-15T02:59:13.085Z 17 340-359 33 33 33 63 63 63 F F F
46322024-02-15T02:59:13.085Z 18 360-379 40 40 40 77 77 77 F F F
46332024-02-15T02:59:13.085Z 19 380-399 41 41 41 78 78 78 F F F
46342024-02-15T02:59:13.085Z 20 400-419 41 41 41 78 78 78 F F F
46352024-02-15T02:59:13.085Z 21 420-439 36 36 36 70 70 70 F F F
46362024-02-15T02:59:13.085Z 22 440-459 40 40 40 75 75 75 F F F
46372024-02-15T02:59:13.085Z 23 460-479 37 37 37 72 72 72 F F F
46382024-02-15T02:59:13.086Z 24 480-499 40 40 40 77 77 77 F F F
46392024-02-15T02:59:13.086Z 25 500-519 40 40 40 76 76 76 F F F
46402024-02-15T02:59:13.086Z 26 520-539 37 37 37 72 72 72 F F F
46412024-02-15T02:59:13.086Z 27 540-559 41 41 41 78 78 78 F F F
46422024-02-15T02:59:13.086Z 28 560-579 41 41 41 78 78 78 F F F
46432024-02-15T02:59:13.086Z 29 580-599 41 41 41 78 78 78 F F F
46442024-02-15T02:59:13.086ZMax gen: 41, Max flush: 78
46452024-02-15T02:59:13.086Z
46462024-02-15T02:59:13.086ZBegin loop 21
46472024-02-15T02:59:13.086ZDownstairs to restart: 1
46482024-02-15T02:59:13.086Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277167323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303}
46492024-02-15T02:59:13.087Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
46502024-02-15T02:59:13.087Z,"time":"2024-02-15T02:59:11.27751094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303}
46512024-02-15T02:59:13.087Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277565573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303}
46522024-02-15T02:59:13.087Z{"msg":"Crucible 89e1f8f1-8cc9-4af1-9dfe-efaa6ee2ad74 has session id: 3b7934ab-59db-430a-8d73-6eb0aa8f85de","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277601509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46532024-02-15T02:59:13.087Z{"msg":"Upstairs opts: Upstairs UUID: 89e1f8f1-8cc9-4af1-9dfe-efaa6ee2ad74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277648083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46542024-02-15T02:59:13.087Z{"msg":"Crucible stats registered with UUID: 89e1f8f1-8cc9-4af1-9dfe-efaa6ee2ad74","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277679519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46552024-02-15T02:59:13.087Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277709705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303}
46562024-02-15T02:59:13.088Z{"msg":"89e1f8f1-8cc9-4af1-9dfe-efaa6ee2ad74 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277738862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46572024-02-15T02:59:13.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277769458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46582024-02-15T02:59:13.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277803903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46592024-02-15T02:59:13.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.27783092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46602024-02-15T02:59:13.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277863946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46612024-02-15T02:59:13.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277902021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46622024-02-15T02:59:13.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277945725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46632024-02-15T02:59:13.088Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.277980391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46642024-02-15T02:59:13.088Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278011977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46652024-02-15T02:59:13.088Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278041444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46662024-02-15T02:59:13.089Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:11.278084998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46672024-02-15T02:59:13.089Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:11.278140781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46682024-02-15T02:59:13.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278184935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46692024-02-15T02:59:13.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278221961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46702024-02-15T02:59:13.089Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:11.278249207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46712024-02-15T02:59:13.090Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278293602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46722024-02-15T02:59:13.090Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278332497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46732024-02-15T02:59:13.090Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.27839181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46742024-02-15T02:59:13.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278923333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46752024-02-15T02:59:13.090Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278960288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46762024-02-15T02:59:13.090Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.278997214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46772024-02-15T02:59:13.090Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.279014602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46782024-02-15T02:59:13.091Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.279546345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46792024-02-15T02:59:13.091Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.279660801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46802024-02-15T02:59:13.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.280040353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46812024-02-15T02:59:13.091Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.280086657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46822024-02-15T02:59:13.091Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.280130952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46832024-02-15T02:59:13.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.280172386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46842024-02-15T02:59:13.091Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.280204673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46852024-02-15T02:59:13.092Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:11.2802276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46862024-02-15T02:59:13.341Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.279971165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46872024-02-15T02:59:13.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.280052155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46882024-02-15T02:59:13.353Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.280823418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46892024-02-15T02:59:13.353Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.280872182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46902024-02-15T02:59:13.353Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.281376389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46912024-02-15T02:59:13.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.2820018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46922024-02-15T02:59:13.353Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282049065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46932024-02-15T02:59:13.353Z{"msg":"[0]R flush_numbers[0..12]: [25, 53, 78, 77, 73, 69, 62, 77, 66, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282097999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46942024-02-15T02:59:13.353Z{"msg":"[0]R generation[0..12]: [14, 27, 41, 40, 38, 36, 32, 40, 34, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282140783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46952024-02-15T02:59:13.353Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282173379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46962024-02-15T02:59:13.353Z{"msg":"[1]R flush_numbers[0..12]: [25, 53, 78, 77, 73, 69, 62, 77, 66, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282192497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46972024-02-15T02:59:13.354Z{"msg":"[1]R generation[0..12]: [14, 27, 41, 40, 38, 36, 32, 40, 34, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282227942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,""Wait for a query_work_queue command to finish before sending IO
46982024-02-15T02:59:13.354Z:"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
46992024-02-15T02:59:13.354Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282277056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47002024-02-15T02:59:13.354Z{"msg":"[2]R flush_numbers[0..12]: [25, 53, 78, 77, 73, 69, 62, 77, 66, 69, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282319191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47012024-02-15T02:59:13.354Z{"msg":"[2]R generation[0..12]: [14, 27, 41, 40, 38, 36, 32, 40, 34, 36, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282347897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47022024-02-15T02:59:13.354Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282369745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47032024-02-15T02:59:13.354Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282412349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47042024-02-15T02:59:13.354Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282449484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47052024-02-15T02:59:13.354Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282465713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47062024-02-15T02:59:13.355Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282489899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47072024-02-15T02:59:13.355Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282524745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47082024-02-15T02:59:13.355Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282556961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47092024-02-15T02:59:13.355Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282573239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47102024-02-15T02:59:13.355Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282597536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47112024-02-15T02:59:13.355Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282622173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47122024-02-15T02:59:13.355Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282654909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47132024-02-15T02:59:13.355Z{"msg":"89e1f8f1-8cc9-4af1-9dfe-efaa6ee2ad74 is now active with session: 3b7934ab-59db-430a-8d73-6eb0aa8f85de","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282670527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47142024-02-15T02:59:13.355Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282694754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303,"session_id":"3b7934ab-59db-430a-8d73-6eb0aa8f85de"}
47152024-02-15T02:59:13.355Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:12.282729539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303}
47162024-02-15T02:59:13.355ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47172024-02-15T02:59:13.355ZLoading write count information from file "/tmp/test_repair_verify.data"
47182024-02-15T02:59:13.355ZRun Repair workload
47192024-02-15T02:59:13.356Z01/30 Write block 286 len 36864 data: 5 5 3 3 5 4 4 4 5
47202024-02-15T02:59:13.356Z02/30 Write block 346 len 4096 data: 5
47212024-02-15T02:59:13.356Z03/30 Read block 340 len 12288
47222024-02-15T02:59:13.356Z04/30 Write block 465 len 36864 data: 5 3 3 4 5 6 6 5 4
47232024-02-15T02:59:13.356Z05/30 Read block 561 len 36864
47242024-02-15T02:59:13.370Z06/30 Write block 553 len 24576 data: 2 2 3 4 6 6
47252024-02-15T02:59:13.371Z07/30 Read block 230 len 8192
47262024-02-15T02:59:13.371Z08/30 Write block 466 len 20480 data: 4 4 5 6 7
47272024-02-15T02:59:13.371Z09/30 Write block 364 len 20480 data: 4 4 5 5 4
47282024-02-15T02:59:13.371Z10/30 Write block 500 len 16384 data: 6 7 8 7
47292024-02-15T02:59:13.371Z11/30 Write block 53 len 16384 data: 3 3 3 3
47302024-02-15T02:59:13.371Z12/30 Read block 586 len 36864
47312024-02-15T02:59:13.371Z13/30 Read block 495 len 24576
47322024-02-15T02:59:13.372Z14/30 Write block 113 len 4096 data: 4
47332024-02-15T02:59:13.372Z15/30 Write block 332 len 4096 data: 6
47342024-02-15T02:59:13.372Z16/30 Flush
47352024-02-15T02:59:13.372Z17/30 Write block 391 len 28672 data: 5 6 5 4 5 4 4
47362024-02-15T02:59:13.372Z18/30 Write block 437 len 32768 data: 3 3 4 4 5 6 7 6
47372024-02-15T02:59:13.372Z19/30 Read block 354 len 12288
47382024-02-15T02:59:13.372Z20/30 Read block 542 len 28672
47392024-02-15T02:59:13.372Z21/30 Read block 419 len 32768
47402024-02-15T02:59:13.373Z22/30 Write block 231 len 40960 data: 5 6 6 6 6 7 5 3 3 3
47412024-02-15T02:59:13.373Z23/30 Write block 267 len 16384 data: 4 5 5 4
47422024-02-15T02:59:13.373Z24/30 Read block 82 len 32768
47432024-02-15T02:59:13.373Z25/30 Write block 481 len 16384 data: 3 3 2 3
47442024-02-15T02:59:13.373Z26/30 Write block 3 len 32768 data: 3 4 4 4 4 5 5 6
47452024-02-15T02:59:13.373Z27/30 Write block 138 len 32768 data: 4 4 6 6 5 5 4 5
47462024-02-15T02:59:13.382Z28/30 Read block 235 len 36864
47472024-02-15T02:59:13.382Z29/30 Write block 435 len 16384 data: 3 3 4 4
47482024-02-15T02:59:13.382Z30/30 Read block 331 len 28672
47492024-02-15T02:59:13.382Z----------------------------------------------------------------
47502024-02-15T02:59:13.382Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
47512024-02-15T02:59:13.382ZStates: Active Active Active
47522024-02-15T02:59:13.382ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
47532024-02-15T02:59:13.382Z 1 Acked 1000 Write 9 Done Done Done false
47542024-02-15T02:59:13.382Z 2 Acked 1001 Write 1 Done Done Done false
47552024-02-15T02:59:13.382Z 3 Acked 1002 Read 3 Done Done Done false
47562024-02-15T02:59:13.383Z 4 Acked 1003 Write 9 Done Sent Done false
47572024-02-15T02:59:13.383Z 5 Acked 1004 Read 9 Done Sent Done false
47582024-02-15T02:59:13.383Z 6 Acked 1005 Write 6 Done Sent Done false
47592024-02-15T02:59:13.383Z 7 Acked 1006 Read 2 Done Sent Done false
47602024-02-15T02:59:13.383Z 8 Acked 1007 Write 5 Done Sent Done false
47612024-02-15T02:59:13.383Z 9 Acked 1008 Write 5 Done Sent Done false
47622024-02-15T02:59:13.383Z 10 Acked 1009 Write 4 Done Sent Done false
47632024-02-15T02:59:13.383Z 11 Acked 1010 Write 4 Done Sent Done false
47642024-02-15T02:59:13.384Z 12 Acked 1011 Read 9 Done Sent Done false
47652024-02-15T02:59:13.384Z 13 Acked 1012 Read 6 Done Sent Done false
47662024-02-15T02:59:13.384Z 14 Acked 1013 Write 1 Done Sent Done false
47672024-02-15T02:59:13.384Z 15 Acked 1014 Write 1 Done Sent Done false
47682024-02-15T02:59:13.384Z 16 Acked 1015 Flush 0 Done Sent Done false
47692024-02-15T02:59:13.384Z 17 Acked 1016 Write 7 Done Sent Done false
47702024-02-15T02:59:13.384Z 18 Acked 1017 Write 8 Done Sent Done false
47712024-02-15T02:59:13.384Z 19 Acked 1018 Read 3 Done Sent Done false
47722024-02-15T02:59:13.384Z 20 Acked 1019 Read 7 Done Sent Done false
47732024-02-15T02:59:13.384Z 21 Acked 1020 Read 8 Done Sent Done false
47742024-02-15T02:59:13.384Z 22 Acked 1021 Write 10 Done Sent Done false
47752024-02-15T02:59:13.384Z 23 Acked 1022 Write 4 Done Sent Done false
47762024-02-15T02:59:13.384Z 24 Acked 1023 Read 8 Done Sent Done false
47772024-02-15T02:59:13.384Z 25 Acked 1024 Write 4 Done Sent Done false
47782024-02-15T02:59:13.385Z 26 Acked 1025 Write 8 Done Sent Done false
47792024-02-15T02:59:13.385Z 27 Acked 1026 Write 8 Done Sent Done false
47802024-02-15T02:59:13.385Z 28 Acked 1027 Read 9 Done Sent Done false
47812024-02-15T02:59:13.385Z 29 Acked 1028 Write 4 Done Sent Done false
47822024-02-15T02:59:13.385Z 30 Acked 1029 Read 7 Done Sent Sent false
47832024-02-15T02:59:13.385Z STATES DS:0 DS:1 DS:2 TOTAL
47842024-02-15T02:59:13.385Z New 0 0 0 0
47852024-02-15T02:59:13.385Z Sent 0 27 1 28
47862024-02-15T02:59:13.385Z Done 30 3 29 62
47872024-02-15T02:59:13.385Z Skipped 0 0 0 0
47882024-02-15T02:59:13.385Z Error 0 0 0 0
47892024-02-15T02:59:13.385ZLast Flush: 1015 0 1015
47902024-02-15T02:59:13.385ZDownstairs last five completed:
47912024-02-15T02:59:13.385ZUpstairs last five completed: 30 29 28 27 26
47922024-02-15T02:59:13.386Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:12.31368635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1303}
47932024-02-15T02:59:13.386ZWrote out file "/tmp/test_repair_verify.data"
47942024-02-15T02:59:13.386Z
47952024-02-15T02:59:13.395ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
47962024-02-15T02:59:13.406ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47972024-02-15T02:59:13.406Z 0 000-019 14 14 14 25 25 25 T F T <---
47982024-02-15T02:59:13.407Z 1 020-039 27 27 27 53 53 53 F F F
47992024-02-15T02:59:13.407Z 2 040-059 42 41 42 79 78 79 F F F <---
48002024-02-15T02:59:13.408Z 3 060-079 40 40 40 77 77 77 F F F
48012024-02-15T02:59:13.408Z 4 080-099 38 38 38 73 73 73 F F F
48022024-02-15T02:59:13.408Z 5 100-119 42 36 42 79 69 79 F F F <---
48032024-02-15T02:59:13.408Z 6 120-139 32 32 32 62 62 62 T F T <---
48042024-02-15T02:59:13.409Z 7 140-159 40 40 40 77 77 77 T F T <---
48052024-02-15T02:59:13.409Z 8 160-179 34 34 34 66 66 66 F F F
48062024-02-15T02:59:13.409Z 9 180-199 36 36 36 69 69 69 F F F
48072024-02-15T02:59:13.410Z 10 200-219 39 39 39 74 74 74 F F F
48082024-02-15T02:59:13.410Z 11 220-239 39 39 39 74 74 74 T F T <---
48092024-02-15T02:59:13.411Z 12 240-259 41 41 41 78 78 78 T F T <---
48102024-02-15T02:59:13.411Z 13 260-279 37 37 37 72 72 72 T F T <---
48112024-02-15T02:59:13.411Z 14 280-299 42 40 42 79 77 79 F T F <---
48122024-02-15T02:59:13.411Z 15 300-319 40 40 40 77 77 77 F F F
48132024-02-15T02:59:13.411Z 16 320-339 42 40 42 79 77 79 F F F <---
48142024-02-15T02:59:13.412Z 17 340-359 42 33 42 79 63 79 F T F <---
48152024-02-15T02:59:13.412Z 18 360-379 42 40 42 79 77 79 F F F <---
48162024-02-15T02:59:13.412Z 19 380-399 41 41 41 78 78 78 T F T <---
48172024-02-15T02:59:13.412Z 20 400-419 41 41 41 78 78 78 F F F
48182024-02-15T02:59:13.412Z 21 420-439 36 36 36 70 70 70 T F T <---
48192024-02-15T02:59:13.412Z 22 440-459 40 40 40 75 75 75 T F T <---
48202024-02-15T02:59:13.413Z 23 460-479 42 37 42 79 72 79 F F F <---
48212024-02-15T02:59:13.413Z 24 480-499 40 40 40 77 77 77 T F T <---
48222024-02-15T02:59:13.413Z 25 500-519 42 40 42 79 76 79 F F F <---
48232024-02-15T02:59:13.413Z 26 520-539 37 37 37 72 72 72 F F F
48242024-02-15T02:59:13.413Z 27 540-559 42 41 42 79 78 79 F F F <---
48252024-02-15T02:59:13.413Z 28 560-579 41 41 41 78 78 78 F F F
48262024-02-15T02:59:13.413Z 29 580-599 41 41 41 78 78 78 F F F
48272024-02-15T02:59:13.413ZMax gen: 42, Max flush: 79
48282024-02-15T02:59:13.413ZError: Difference in extent metadata found!
48292024-02-15T02:59:13.426ZA Difference in extent metadata is expected here
48302024-02-15T02:59:13.426ZOn loop 21
48312024-02-15T02:59:13.426Z
48322024-02-15T02:59:13.426ZVerifying data now
48332024-02-15T02:59:19.335ZNo change after verify
48342024-02-15T02:59:19.335ZLoop: 21 Downstairs dump after verify (and repair):
48352024-02-15T02:59:20.706ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48362024-02-15T02:59:20.706Z 0 000-019 43 43 43 80 80 80 F F F
48372024-02-15T02:59:20.706Z 1 020-039 27 27 27 53 53 53 F F F
48382024-02-15T02:59:20.706Z 2 040-059 42 42 42 79 79 79 F F F
48392024-02-15T02:59:20.706Z 3 060-079 40 40 40 77 77 77 F F F
48402024-02-15T02:59:20.706Z 4 080-099 38 38 38 73 73 73 F F F
48412024-02-15T02:59:20.706Z 5 100-119 42 42 42 79 79 79 F F F
48422024-02-15T02:59:20.706Z 6 120-139 43 43 43 80 80 80 F F F
48432024-02-15T02:59:20.706Z 7 140-159 43 43 43 80 80 80 F F F
48442024-02-15T02:59:20.706Z 8 160-179 34 34 34 66 66 66 F F F
48452024-02-15T02:59:21.574Z 9 180-199 36 36 36 69 69 69 F F F
48462024-02-15T02:59:21.574Z 10 200-219 39 39 39 74 74 74 F F F
48472024-02-15T02:59:21.574Z 11 220-239 43 43 43 80 80 80 F F F
48482024-02-15T02:59:21.574Z 12 240-259 43 43 43 80 80 80 F F F
48492024-02-15T02:59:21.574Z 13 260-279 43 43 43 80 80 80 F F F
48502024-02-15T02:59:21.574Z 14 280-299 42 42 42 79 79 79 F F F
48512024-02-15T02:59:21.575Z 15 300-319 40 40 40 77 77 77 F F F
48522024-02-15T02:59:21.575Z 16 320-339 42 42 42 79 79 79 F F F
48532024-02-15T02:59:21.575Z 17 340-359 42 42 42 79 79 79 F F F
48542024-02-15T02:59:21.575Z 18 360-379 42 42 42 79 79 79 F F F
48552024-02-15T02:59:21.575Z 19 380-399 43 43 43 80 80 80 F F F
48562024-02-15T02:59:21.575Z 20 400-419 41 41 41 78 78 78 F F F
48572024-02-15T02:59:21.575Z 21 420-439 43 43 43 80 80 80 F F F
48582024-02-15T02:59:21.575Z 22 440-459 43 43 43 80 80 80 F F F
48592024-02-15T02:59:21.575Z 23 460-479 42 42 42 79 79 79 F F F
48602024-02-15T02:59:21.575Z 24 480-499 43 43 43 80 80 80 F F F
48612024-02-15T02:59:21.576Z 25 500-519 42 42 42 79 79 79 F F F
48622024-02-15T02:59:21.576Z 26 520-539 37 37 37 72 72 72 F F F
48632024-02-15T02:59:21.576Z 27 540-559 42 42 42 79 79 79 F F F
48642024-02-15T02:59:21.576Z 28 560-579 41 41 41 78 78 78 F F F
48652024-02-15T02:59:21.576Z 29 580-599 41 41 41 78 78 78 F F F
48662024-02-15T02:59:21.576ZMax gen: 43, Max flush: 80
48672024-02-15T02:59:21.576Z
48682024-02-15T02:59:21.576ZBegin loop 22
48692024-02-15T02:59:21.576ZDownstairs to restart: 1
48702024-02-15T02:59:21.576Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308120036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311}
48712024-02-15T02:59:21.576Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308365645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311}
48722024-02-15T02:59:21.576Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308382753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311}
48732024-02-15T02:59:21.577Z{"msg":"Crucible 159c4a5d-80e6-4cc4-b27a-24e95f32bdcd has session id: c20edd53-a754-4c2b-a832-0ae74c4349e1","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308421558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48742024-02-15T02:59:21.577Z{Crucible runtime is spawned
48752024-02-15T02:59:21.577Z"msg":"Upstairs opts: Upstairs UUID: 159c4a5d-80e6-4cc4-b27a-24e95f32bdcd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308476821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48762024-02-15T02:59:21.577Z{"msg":"Crucible stats registered with UUID: 159c4a5d-80e6-4cc4-b27a-24e95f32bdcd","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308523995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48772024-02-15T02:59:21.577Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308552372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311}
48782024-02-15T02:59:21.577Z{"msg":"159c4a5d-80e6-4cc4-b27a-24e95f32bdcd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.30864474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48792024-02-15T02:59:21.577Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308682176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48802024-02-15T02:59:21.577Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308710072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48812024-02-15T02:59:21.577Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308748438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48822024-02-15T02:59:21.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308775164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48832024-02-15T02:59:21.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308810859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48842024-02-15T02:59:21.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308829917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48852024-02-15T02:59:21.578Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308847065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48862024-02-15T02:59:21.578Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.30888654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48872024-02-15T02:59:21.578Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.308913957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48882024-02-15T02:59:21.578Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:18.308945083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48892024-02-15T02:59:21.578Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:18.308975559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48902024-02-15T02:59:21.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.309016764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48912024-02-15T02:59:21.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.309040911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48922024-02-15T02:59:21.578Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:18.309075457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48932024-02-15T02:59:21.579Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.309099363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48942024-02-15T02:59:21.579Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.309133219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48952024-02-15T02:59:21.579Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.309349112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48962024-02-15T02:59:21.579Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.30992953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48972024-02-15T02:59:21.579Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.309971574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48982024-02-15T02:59:21.580Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.309999661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
48992024-02-15T02:59:21.580Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.310036276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49002024-02-15T02:59:21.580Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.310549902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49012024-02-15T02:59:21.580Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.310577728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49022024-02-15T02:59:21.580Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.311138628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49032024-02-15T02:59:21.580Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.31119947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49042024-02-15T02:59:21.580Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.311241385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49052024-02-15T02:59:21.580Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.311261583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49062024-02-15T02:59:21.580Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.311291749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49072024-02-15T02:59:21.581Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:18.311311706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49082024-02-15T02:59:21.581Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.310999858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49092024-02-15T02:59:21.581Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.311206492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49102024-02-15T02:59:21.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.312116678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49112024-02-15T02:59:21.581Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.312154074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49122024-02-15T02:59:21.581Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.312668549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49132024-02-15T02:59:21.581Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313242207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49142024-02-15T02:59:21.581Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313276643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49152024-02-15T02:59:21.581Z{"msg":"[0]R flush_numbers[0..12]: [80, 53, 79, 77, 73, 79, 80, 80, 66, 69, 74, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313295391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49162024-02-15T02:59:21.581Z{"msg":"[0]R generation[0..12]: [43, 27, 42, 40, 38, 42, 43, 43, 34, 36, 39, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313312968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49172024-02-15T02:59:21.582Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313331046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49182024-02-15T02:59:21.582Z{"msg":"[1]R flush_numbers[0..12]: [80, 53, 79, 77, 73, 79, 80, 80, 66, 69, 74, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313357953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49192024-02-15T02:59:21.582Z{"msg":"[1]R generation[0..12]: [43, 27, 42, 40, 38, 42, 43, 43, 34, 36, 39, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.31337583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49202024-02-15T02:59:21.582Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313402127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49212024-02-15T02:59:21.582Z{"msg":"[2]R flush_numbers[0..12]: [80, 53, 79, 77, 73, 79, 80, 80, 66, 69, 74, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313419975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49222024-02-15T02:59:21.582Z{"msg":"[2]R generation[0..12]: [43, 27, 42, 40, 38, 42, 43, 43, 34, 36, 39, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313447201Z"Wait for a query_work_queue command to finish before sending IO
49232024-02-15T02:59:21.582Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49242024-02-15T02:59:21.582Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313470338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49252024-02-15T02:59:21.582Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313495785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49262024-02-15T02:59:21.582Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313513413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49272024-02-15T02:59:21.583Z{"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313545539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49282024-02-15T02:59:21.583Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313572606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49292024-02-15T02:59:21.583Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.31361852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49302024-02-15T02:59:21.583Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313656605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49312024-02-15T02:59:21.583Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313686362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49322024-02-15T02:59:21.583Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313718727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49332024-02-15T02:59:21.583Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313753853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49342024-02-15T02:59:21.583Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.31377893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49352024-02-15T02:59:21.583Z{"msg":"159c4a5d-80e6-4cc4-b27a-24e95f32bdcd is now active with session: c20edd53-a754-4c2b-a832-0ae74c4349e1","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313793028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49362024-02-15T02:59:21.583Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313817295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311,"session_id":"c20edd53-a754-4c2b-a832-0ae74c4349e1"}
49372024-02-15T02:59:21.583Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:19.313851521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311}
49382024-02-15T02:59:21.584ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
49392024-02-15T02:59:21.584ZLoading write count information from file "/tmp/test_repair_verify.data"
49402024-02-15T02:59:21.584ZRun Repair workload
49412024-02-15T02:59:21.584Z01/30 Write block 381 len 20480 data: 5 5 5 6 4
49422024-02-15T02:59:21.584Z02/30 Read block 6 len 36864
49432024-02-15T02:59:21.584Z03/30 Write block 290 len 36864 data: 6 5 5 5 6 4 5 4 5
49442024-02-15T02:59:21.584Z04/30 Read block 379 len 16384
49452024-02-15T02:59:21.607Z05/30 Read block 29 len 40960
49462024-02-15T02:59:21.607Z06/30 Flush
49472024-02-15T02:59:21.608Z07/30 Flush
49482024-02-15T02:59:21.608Z08/30 Write block 206 len 40960 data: 4 4 4 3 3 4 4 4 5 4
49492024-02-15T02:59:21.608Z09/30 Write block 265 len 8192 data: 7 6
49502024-02-15T02:59:21.608Z10/30 Write block 517 len 20480 data: 3 3 2 2 2
49512024-02-15T02:59:21.608Z11/30 Write block 326 len 36864 data: 6 5 5 5 5 6 7 6 6
49522024-02-15T02:59:21.609Z12/30 Write block 75 len 4096 data: 4
49532024-02-15T02:59:21.609Z13/30 Read block 487 len 40960
49542024-02-15T02:59:21.609Z14/30 Read block 523 len 12288
49552024-02-15T02:59:21.609Z15/30 Read block 486 len 40960
49562024-02-15T02:59:21.609Z16/30 Write block 434 len 32768 data: 2 4 4 5 5 5 5 6
49572024-02-15T02:59:21.609Z17/30 Write block 117 len 32768 data: 5 4 4 4 5 4 4 3
49582024-02-15T02:59:21.609Z18/30 Flush
49592024-02-15T02:59:21.609Z19/30 Write block 563 len 4096 data: 5
49602024-02-15T02:59:21.609Z20/30 Read block 252 len 8192
49612024-02-15T02:59:21.610Z21/30 Read block 359 len 8192
49622024-02-15T02:59:21.610Z22/30 Read block 72 len 20480
49632024-02-15T02:59:21.610Z23/30 Write block 543 len 4096 data: 6
49642024-02-15T02:59:21.610Z24/30 Read block 263 len 32768
49652024-02-15T02:59:21.610Z25/30 Write block 157 len 4096 data: 4
49662024-02-15T02:59:21.610Z26/30 Read block 501 len 4096
49672024-02-15T02:59:21.610Z27/30 Read block 233 len 28672
49682024-02-15T02:59:21.610Z28/30 Read block 403 len 8192
49692024-02-15T02:59:21.611Z29/30 Write block 481 len 24576 data: 4 4 3 4 3 4
49702024-02-15T02:59:21.611Z30/30 Write block 97 len 40960 data: 9 9 7 6 7 7 5 5 6 4
49712024-02-15T02:59:21.611Z----------------------------------------------------------------
49722024-02-15T02:59:21.611Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
49732024-02-15T02:59:21.611ZStates: Active Active Active
49742024-02-15T02:59:21.611ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
49752024-02-15T02:59:21.611Z 1 Acked 1000 Write 5 Done Done Done false
49762024-02-15T02:59:21.612Z 2 Acked 1001 Read 9 Done Done Done false
49772024-02-15T02:59:21.612Z 3 Acked 1002 Write 9 Done Done Done false
49782024-02-15T02:59:21.612Z 4 Acked 1003 Read 4 Done Done Done false
49792024-02-15T02:59:21.612Z 5 Acked 1004 Read 10 Done Done Done false
49802024-02-15T02:59:21.612Z 6 Acked 1005 Flush 0 Done Sent Done false
49812024-02-15T02:59:21.612Z 7 Acked 1006 Flush 0 Done Sent Done false
49822024-02-15T02:59:21.612Z 8 Acked 1007 Write 10 Done Sent Done false
49832024-02-15T02:59:21.612Z 9 Acked 1008 Write 2 Done Sent Done false
49842024-02-15T02:59:21.612Z 10 Acked 1009 Write 5 Done Sent Done false
49852024-02-15T02:59:21.613Z 11 Acked 1010 Write 9 Done Sent Done false
49862024-02-15T02:59:21.613Z 12 Acked 1011 Write 1 Done Sent Done false
49872024-02-15T02:59:21.613Z 13 Acked 1012 Read 10 Done Sent Done false
49882024-02-15T02:59:21.613Z 14 Acked 1013 Read 3 Done Sent Done false
49892024-02-15T02:59:21.613Z 15 Acked 1014 Read 10 Done Sent Done false
49902024-02-15T02:59:21.613Z 16 Acked 1015 Write 8 Done Sent Done false
49912024-02-15T02:59:21.613Z 17 Acked 1016 Write 8 Done Sent Done false
49922024-02-15T02:59:21.614Z 18 Acked 1017 Flush 0 Done Sent Done false
49932024-02-15T02:59:21.614Z 19 Acked 1018 Write 1 Done Sent Done false
49942024-02-15T02:59:21.614Z 20 Acked 1019 Read 2 Done Sent Done false
49952024-02-15T02:59:21.614Z 21 Acked 1020 Read 2 Done Sent Done false
49962024-02-15T02:59:21.614Z 22 Acked 1021 Read 5 Done Sent Done false
49972024-02-15T02:59:21.614Z 23 Acked 1022 Write 1 Done Sent Done false
49982024-02-15T02:59:21.614Z 24 Acked 1023 Read 8 Done Sent Done false
49992024-02-15T02:59:21.615Z 25 Acked 1024 Write 1 Done Sent Done false
50002024-02-15T02:59:21.615Z 26 Acked 1025 Read 1 Done Sent Done false
50012024-02-15T02:59:21.615Z 27 Acked 1026 Read 7 Done Sent Done false
50022024-02-15T02:59:21.615Z 28 Acked 1027 Read 2 Done Sent Done false
50032024-02-15T02:59:21.615Z 29 Acked 1028 Write 6 Done Sent Done false
50042024-02-15T02:59:21.615Z 30 Acked 1029 Write 10 Sent Sent Sent false
50052024-02-15T02:59:21.615Z STATES DS:0 DS:1 DS:2 TOTAL
50062024-02-15T02:59:21.615Z New 0 0 0 0
50072024-02-15T02:59:21.615Z Sent 1 25 1 27
50082024-02-15T02:59:21.616Z Done 29 5 29 63
50092024-02-15T02:59:21.616Z Skipped 0 0 0 0
50102024-02-15T02:59:21.616Z Error 0 0 0 0
50112024-02-15T02:59:21.616ZLast Flush: 1017 0 1017
50122024-02-15T02:59:21.616ZDownstairs last five completed:
50132024-02-15T02:59:21.616ZUpstairs last five completed: 30 29 28 27 26
50142024-02-15T02:59:21.616Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:19.344950894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1311}
50152024-02-15T02:59:21.616ZWrote out file "/tmp/test_repair_verify.data"
50162024-02-15T02:59:21.617Z
50172024-02-15T02:59:21.617ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
50182024-02-15T02:59:21.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50192024-02-15T02:59:21.617Z 0 000-019 43 43 43 80 80 80 F F F
50202024-02-15T02:59:21.617Z 1 020-039 27 27 27 53 53 53 F F F
50212024-02-15T02:59:21.617Z 2 040-059 42 42 42 79 79 79 F F F
50222024-02-15T02:59:21.617Z 3 060-079 44 40 44 83 77 83 F F F <---
50232024-02-15T02:59:21.617Z 4 080-099 38 38 38 73 73 73 T F T <---
50242024-02-15T02:59:21.618Z 5 100-119 44 42 44 83 79 83 T F T <---
50252024-02-15T02:59:21.618Z 6 120-139 44 43 44 83 80 83 F F F <---
50262024-02-15T02:59:21.618Z 7 140-159 43 43 43 80 80 80 T F T <---
50272024-02-15T02:59:21.618Z 8 160-179 34 34 34 66 66 66 F F F
50282024-02-15T02:59:21.618Z 9 180-199 36 36 36 69 69 69 F F F
50292024-02-15T02:59:21.618Z 10 200-219 44 39 44 83 74 83 F F F <---
50302024-02-15T02:59:21.619Z 11 220-239 43 43 43 80 80 80 F F F
50312024-02-15T02:59:21.619Z 12 240-259 43 43 43 80 80 80 F F F
50322024-02-15T02:59:21.619Z 13 260-279 44 43 44 83 80 83 F F F <---
50332024-02-15T02:59:21.619Z 14 280-299 44 42 44 81 79 81 F T F <---
50342024-02-15T02:59:21.619Z 15 300-319 40 40 40 77 77 77 F F F
50352024-02-15T02:59:21.619Z 16 320-339 44 42 44 83 79 83 F F F <---
50362024-02-15T02:59:21.619Z 17 340-359 42 42 42 79 79 79 F F F
50372024-02-15T02:59:21.620Z 18 360-379 42 42 42 79 79 79 F F F
50382024-02-15T02:59:21.620Z 19 380-399 44 43 44 81 80 81 F T F <---
50392024-02-15T02:59:21.620Z 20 400-419 41 41 41 78 78 78 F F F
50402024-02-15T02:59:21.620Z 21 420-439 44 43 44 83 80 83 F F F <---
50412024-02-15T02:59:21.620Z 22 440-459 44 43 44 83 80 83 F F F <---
50422024-02-15T02:59:21.620Z 23 460-479 42 42 42 79 79 79 F F F
50432024-02-15T02:59:21.621Z 24 480-499 43 43 43 80 80 80 T F T <---
50442024-02-15T02:59:21.621Z 25 500-519 44 42 44 83 79 83 F F F <---
50452024-02-15T02:59:21.633Z 26 520-539 44 37 44 83 72 83 F F F <---
50462024-02-15T02:59:21.634Z 27 540-559 42 42 42 79 79 79 T F T <---
50472024-02-15T02:59:21.634Z 28 560-579 41 41 41 78 78 78 T F T <---
50482024-02-15T02:59:21.634Z 29 580-599 41 41 41 78 78 78 F F F
50492024-02-15T02:59:21.634ZMax gen: 44, Max flush: 83
50502024-02-15T02:59:21.634ZError: Difference in extent metadata found!
50512024-02-15T02:59:21.634ZA Difference in extent metadata is expected here
50522024-02-15T02:59:21.635ZOn loop 22
50532024-02-15T02:59:21.635Z
50542024-02-15T02:59:21.635ZVerifying data now
50552024-02-15T02:59:26.263ZNo change after verify
50562024-02-15T02:59:26.264ZLoop: 22 Downstairs dump after verify (and repair):
50572024-02-15T02:59:26.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50582024-02-15T02:59:26.285Z 0 000-019 43 43 43 80 80 80 F F F
50592024-02-15T02:59:26.286Z 1 020-039 27 27 27 53 53 53 F F F
50602024-02-15T02:59:26.286Z 2 040-059 42 42 42 79 79 79 F F F
50612024-02-15T02:59:26.286Z 3 060-079 44 44 44 83 83 83 F F F
50622024-02-15T02:59:26.286Z 4 080-099 45 45 45 84 84 84 F F F
50632024-02-15T02:59:26.286Z 5 100-119 45 45 45 84 84 84 F F F
50642024-02-15T02:59:26.287Z 6 120-139 44 44 44 83 83 83 F F F
50652024-02-15T02:59:26.287Z 7 140-159 45 45 45 84 84 84 F F F
50662024-02-15T02:59:26.287Z 8 160-179 34 34 34 66 66 66 F F F
50672024-02-15T02:59:26.287Z 9 180-199 36 36 36 69 69 69 F F F
50682024-02-15T02:59:26.287Z 10 200-219 44 44 44 83 83 83 F F F
50692024-02-15T02:59:26.287Z 11 220-239 43 43 43 80 80 80 F F F
50702024-02-15T02:59:26.288Z 12 240-259 43 43 43 80 80 80 F F F
50712024-02-15T02:59:26.735Z 13 260-279 44 44 44 83 83 83 F F F
50722024-02-15T02:59:26.735Z 14 280-299 44 44 44 81 81 81 F F F
50732024-02-15T02:59:26.735Z 15 300-319 40 40 40 77 77 77 F F F
50742024-02-15T02:59:26.735Z 16 320-339 44 44 44 83 83 83 F F F
50752024-02-15T02:59:26.735Z 17 340-359 42 42 42 79 79 79 F F F
50762024-02-15T02:59:26.735Z 18 360-379 42 42 42 79 79 79 F F F
50772024-02-15T02:59:26.736Z 19 380-399 44 44 44 81 81 81 F F F
50782024-02-15T02:59:26.736Z 20 400-419 41 41 41 78 78 78 F F F
50792024-02-15T02:59:26.736Z 21 420-439 44 44 44 83 83 83 F F F
50802024-02-15T02:59:26.736Z 22 440-459 44 44 44 83 83 83 F F F
50812024-02-15T02:59:26.736Z 23 460-479 42 42 42 79 79 79 F F F
50822024-02-15T02:59:26.736Z 24 480-499 45 45 45 84 84 84 F F F
50832024-02-15T02:59:26.736Z 25 500-519 44 44 44 83 83 83 F F F
50842024-02-15T02:59:26.736Z 26 520-539 44 44 44 83 83 83 F F F
50852024-02-15T02:59:26.737Z 27 540-559 45 45 45 84 84 84 F F F
50862024-02-15T02:59:26.737Z 28 560-579 45 45 45 84 84 84 F F F
50872024-02-15T02:59:26.737Z 29 580-599 41 41 41 78 78 78 F F F
50882024-02-15T02:59:26.737ZMax gen: 45, Max flush: 84
50892024-02-15T02:59:26.737Z
50902024-02-15T02:59:26.737ZBegin loop 23
50912024-02-15T02:59:26.737ZDownstairs to restart: 0
50922024-02-15T02:59:26.752Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.2400913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319}
50932024-02-15T02:59:26.753Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240344758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319}
50942024-02-15T02:59:26.753Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240367195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319}
50952024-02-15T02:59:26.753Z{"msg":"Crucible 241a8f36-8e57-4628-a218-fa0400604433 has session id: 39870fc2-37c1-4e96-8273-79d16e2001f3","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240400571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
50962024-02-15T02:59:26.753Z{"msg":"Crucible runtime is spawned
50972024-02-15T02:59:26.753ZUpstairs opts: Upstairs UUID: 241a8f36-8e57-4628-a218-fa0400604433, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240441956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
50982024-02-15T02:59:26.753Z{"msg":"Crucible stats registered with UUID: 241a8f36-8e57-4628-a218-fa0400604433","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240477832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
50992024-02-15T02:59:26.753Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240502428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319}
51002024-02-15T02:59:26.753Z{"msg":"241a8f36-8e57-4628-a218-fa0400604433 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240626513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51012024-02-15T02:59:26.753Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.24065176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51022024-02-15T02:59:26.753Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240667248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51032024-02-15T02:59:26.753Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240686145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51042024-02-15T02:59:26.753Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240722281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51052024-02-15T02:59:26.753Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240748978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51062024-02-15T02:59:26.753Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240774694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51072024-02-15T02:59:26.754Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240798281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51082024-02-15T02:59:26.754Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240837496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51092024-02-15T02:59:26.754Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.240863113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51102024-02-15T02:59:26.754Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.241034432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51112024-02-15T02:59:26.754Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.241083965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51122024-02-15T02:59:26.754Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:25.241199981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51132024-02-15T02:59:26.754Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:25.241258284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51142024-02-15T02:59:26.754Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:25.241298849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51152024-02-15T02:59:26.754Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.241330345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51162024-02-15T02:59:26.754Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.24136448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51172024-02-15T02:59:26.754Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.24144372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51182024-02-15T02:59:26.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.241909302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51192024-02-15T02:59:26.754Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.241956606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51202024-02-15T02:59:26.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.24200773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51212024-02-15T02:59:26.754Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.242031207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51222024-02-15T02:59:26.754Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.242515386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51232024-02-15T02:59:26.754Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.242569599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51242024-02-15T02:59:26.754Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.243156426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51252024-02-15T02:59:26.754Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.243209349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51262024-02-15T02:59:26.754Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.243239386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51272024-02-15T02:59:26.754Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.243264722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51282024-02-15T02:59:26.755Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.24328058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51292024-02-15T02:59:26.755Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:25.243295348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51302024-02-15T02:59:27.307Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.242846988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51312024-02-15T02:59:27.334Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.242977062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51322024-02-15T02:59:27.334Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.243887907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51332024-02-15T02:59:27.334Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.243918613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51342024-02-15T02:59:27.335Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.244478183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51352024-02-15T02:59:27.335Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245071609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51362024-02-15T02:59:27.335Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245108314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51372024-02-15T02:59:27.335Z{"msg":"[0]R flush_numbers[0..12]: [80, 53, 79, 83, 84, 84, 83, 84, 66, 69, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245133231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51382024-02-15T02:59:27.335Z{"msg":"[0]R generation[0..12]: [43, 27, 42, 44, 45, 45, 44, 45, 34, 36, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245172186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51392024-02-15T02:59:27.335Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245187854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51402024-02-15T02:59:27.335Z{"msg":"[1]R flush_numbers[0..12]: [80, 53, 79, 83, 84, 84, 83, 84, 66, 69, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245202323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51412024-02-15T02:59:27.336Z{"msg":"[1]R generation[0..12]: [43, 27, 42, 44, 45, 45, 44, 45, 34, 36, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.24522413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51422024-02-15T02:59:27.336Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245239568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51432024-02-15T02:59:27.336Z{"msg":"[2]R flush_numbers[0..12]: [80, 53, 79, 83, 84, 84, 83, 84, 66, 69, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245260275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51442024-02-15T02:59:27.336Z{"msg":"[2]R generation[0..12]: [43, 27, 42, 44, 45, 45, 44, 45, 34, 36, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245274174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51452024-02-15T02:59:27.336Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245295401Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"Wait for a query_work_queue command to finish before sending IO
51462024-02-15T02:59:27.336Zdownstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51472024-02-15T02:59:27.336Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245313129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51482024-02-15T02:59:27.336Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245333596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51492024-02-15T02:59:27.337Z{"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245346765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51502024-02-15T02:59:27.337Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245366712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51512024-02-15T02:59:27.337Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.24538065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51522024-02-15T02:59:27.337Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245393499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51532024-02-15T02:59:27.337Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245413176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51542024-02-15T02:59:27.337Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245426624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51552024-02-15T02:59:27.337Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245451841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51562024-02-15T02:59:27.337Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245470989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51572024-02-15T02:59:27.338Z{"msg":"241a8f36-8e57-4628-a218-fa0400604433 is now active with session: 39870fc2-37c1-4e96-8273-79d16e2001f3","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245495646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51582024-02-15T02:59:27.338Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245509064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319,"session_id":"39870fc2-37c1-4e96-8273-79d16e2001f3"}
51592024-02-15T02:59:27.338Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:26.245529582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319}
51602024-02-15T02:59:27.338ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
51612024-02-15T02:59:27.338ZLoading write count information from file "/tmp/test_repair_verify.data"
51622024-02-15T02:59:27.338ZRun Repair workload
51632024-02-15T02:59:27.338Z01/30 Write block 142 len 12288 data: 6 6 5
51642024-02-15T02:59:27.338Z02/30 Write block 250 len 24576 data: 5 4 5 5 6 5
51652024-02-15T02:59:27.339Z03/30 Flush
51662024-02-15T02:59:27.339Z04/30 Write block 502 len 32768 data: 9 8 7 6 6 4 3 4
51672024-02-15T02:59:27.339Z05/30 Read block 24 len 40960
51682024-02-15T02:59:27.339Z06/30 Read block 250 len 16384
51692024-02-15T02:59:27.339Z07/30 Read block 125 len 12288
51702024-02-15T02:59:27.339Z08/30 Read block 318 len 28672
51712024-02-15T02:59:27.339Z09/30 Read block 56 len 28672
51722024-02-15T02:59:27.359Z10/30 Write block 576 len 8192 data: 7 6
51732024-02-15T02:59:27.360Z11/30 Flush
51742024-02-15T02:59:27.360Z12/30 Write block 512 len 20480 data: 6 6 6 5 5
51752024-02-15T02:59:27.360Z13/30 Write block 189 len 36864 data: 5 5 5 6 8 7 7 5 4
51762024-02-15T02:59:27.360Z14/30 Write block 131 len 16384 data: 4 5 6 6
51772024-02-15T02:59:27.360Z15/30 Read block 240 len 12288
51782024-02-15T02:59:27.361Z16/30 Read block 443 len 12288
51792024-02-15T02:59:27.361Z17/30 Read block 506 len 36864
51802024-02-15T02:59:27.361Z18/30 Write block 240 len 4096 data: 4
51812024-02-15T02:59:27.361Z19/30 Flush
51822024-02-15T02:59:27.361Z20/30 Write block 179 len 16384 data: 4 4 3 3
51832024-02-15T02:59:27.362Z21/30 Read block 357 len 32768
51842024-02-15T02:59:27.362Z22/30 Read block 115 len 16384
51852024-02-15T02:59:27.362Z23/30 Read block 292 len 20480
51862024-02-15T02:59:27.362Z24/30 Write block 18 len 4096 data: 3
51872024-02-15T02:59:27.363Z25/30 Read block 436 len 8192
51882024-02-15T02:59:27.363Z26/30 Read block 95 len 36864
51892024-02-15T02:59:27.364Z27/30 Write block 52 len 24576 data: 2 4 4 4 4 4
51902024-02-15T02:59:27.364Z28/30 Write block 326 len 16384 data: 7 6 6 6
51912024-02-15T02:59:27.364Z29/30 Write block 436 len 12288 data: 5 6 6
51922024-02-15T02:59:27.364Z30/30 Write block 36 len 36864 data: 4 4 5 4 3 3 3 2 5
51932024-02-15T02:59:27.365Z----------------------------------------------------------------
51942024-02-15T02:59:27.365Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
51952024-02-15T02:59:27.365ZStates: Active Active Active
51962024-02-15T02:59:27.366ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
51972024-02-15T02:59:27.366Z 1 Acked 1000 Write 3 Done Done Done false
51982024-02-15T02:59:27.366Z 2 Acked 1001 Write 6 Done Done Done false
51992024-02-15T02:59:27.366Z 3 Acked 1002 Flush 0 Sent Done Done false
52002024-02-15T02:59:27.366Z 4 Acked 1003 Write 8 Sent Done Done false
52012024-02-15T02:59:27.366Z 5 Acked 1004 Read 10 Sent Done Done false
52022024-02-15T02:59:27.367Z 6 Acked 1005 Read 4 Sent Done Done false
52032024-02-15T02:59:27.367Z 7 Acked 1006 Read 3 Sent Done Done false
52042024-02-15T02:59:27.367Z 8 Acked 1007 Read 7 Sent Done Done false
52052024-02-15T02:59:27.367Z 9 Acked 1008 Read 7 Sent Done Done false
52062024-02-15T02:59:27.367Z 10 Acked 1009 Write 2 Sent Done Done false
52072024-02-15T02:59:27.367Z 11 Acked 1010 Flush 0 Sent Done Done false
52082024-02-15T02:59:27.368Z 12 Acked 1011 Write 5 Sent Done Done false
52092024-02-15T02:59:27.368Z 13 Acked 1012 Write 9 Sent Done Done false
52102024-02-15T02:59:27.368Z 14 Acked 1013 Write 4 Sent Done Done false
52112024-02-15T02:59:27.368Z 15 Acked 1014 Read 3 Sent Done Done false
52122024-02-15T02:59:27.368Z 16 Acked 1015 Read 3 Sent Done Done false
52132024-02-15T02:59:27.369Z 17 Acked 1016 Read 9 Sent Done Done false
52142024-02-15T02:59:27.369Z 18 Acked 1017 Write 1 Sent Done Done false
52152024-02-15T02:59:27.369Z 19 Acked 1018 Flush 0 Sent Done Done false
52162024-02-15T02:59:27.369Z 20 Acked 1019 Write 4 Sent Done Done false
52172024-02-15T02:59:27.369Z 21 Acked 1020 Read 8 Sent Done Done false
52182024-02-15T02:59:27.369Z 22 Acked 1021 Read 4 Sent Done Done false
52192024-02-15T02:59:27.369Z 23 Acked 1022 Read 5 Sent Done Done false
52202024-02-15T02:59:27.370Z 24 Acked 1023 Write 1 Sent Done Done false
52212024-02-15T02:59:27.370Z 25 Acked 1024 Read 2 Sent Done Done false
52222024-02-15T02:59:27.370Z 26 Acked 1025 Read 9 Sent Done Done false
52232024-02-15T02:59:27.370Z 27 Acked 1026 Write 6 Sent Done Done false
52242024-02-15T02:59:27.370Z 28 Acked 1027 Write 4 Sent Done Done false
52252024-02-15T02:59:27.370Z 29 Acked 1028 Write 3 Sent Done Done false
52262024-02-15T02:59:27.370Z 30 Acked 1029 Write 9 Sent Sent Sent false
52272024-02-15T02:59:27.370Z STATES DS:0 DS:1 DS:2 TOTAL
52282024-02-15T02:59:27.371Z New 0 0 0 0
52292024-02-15T02:59:27.371Z Sent 28 1 1 30
52302024-02-15T02:59:27.371Z Done 2 29 29 60
52312024-02-15T02:59:27.371Z Skipped 0 0 0 0
52322024-02-15T02:59:27.371Z Error 0 0 0 0
52332024-02-15T02:59:27.371ZLast Flush: 0 1018 1018
52342024-02-15T02:59:27.372ZDownstairs last five completed:
52352024-02-15T02:59:27.372ZUpstairs last five completed: 30 29 28 27 26
52362024-02-15T02:59:27.372Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:26.276449148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1319}
52372024-02-15T02:59:27.372ZWrote out file "/tmp/test_repair_verify.data"
52382024-02-15T02:59:27.372Z
52392024-02-15T02:59:27.372ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
52402024-02-15T02:59:27.379ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
52412024-02-15T02:59:27.379Z 0 000-019 43 43 43 80 80 80 F T T <---
52422024-02-15T02:59:27.379Z 1 020-039 27 27 27 53 53 53 F T T <---
52432024-02-15T02:59:27.379Z 2 040-059 42 42 42 79 79 79 F T T <---
52442024-02-15T02:59:27.380Z 3 060-079 44 44 44 83 83 83 F F F
52452024-02-15T02:59:27.380Z 4 080-099 45 45 45 84 84 84 F F F
52462024-02-15T02:59:27.380Z 5 100-119 45 45 45 84 84 84 F F F
52472024-02-15T02:59:27.380Z 6 120-139 44 46 46 83 87 87 F F F <---
52482024-02-15T02:59:27.380Z 7 140-159 45 46 46 84 85 85 T F F <---
52492024-02-15T02:59:27.380Z 8 160-179 34 34 34 66 66 66 F T T <---
52502024-02-15T02:59:27.380Z 9 180-199 36 46 46 69 87 87 F T T <---
52512024-02-15T02:59:27.381Z 10 200-219 44 44 44 83 83 83 F F F
52522024-02-15T02:59:27.381Z 11 220-239 43 43 43 80 80 80 F F F
52532024-02-15T02:59:27.381Z 12 240-259 43 46 46 80 87 87 T F F <---
52542024-02-15T02:59:27.381Z 13 260-279 44 44 44 83 83 83 F F F
52552024-02-15T02:59:27.381Z 14 280-299 44 44 44 81 81 81 F F F
52562024-02-15T02:59:27.382Z 15 300-319 40 40 40 77 77 77 F F F
52572024-02-15T02:59:27.382Z 16 320-339 44 44 44 83 83 83 F T T <---
52582024-02-15T02:59:27.382Z 17 340-359 42 42 42 79 79 79 F F F
52592024-02-15T02:59:27.382Z 18 360-379 42 42 42 79 79 79 F F F
52602024-02-15T02:59:27.382Z 19 380-399 44 44 44 81 81 81 F F F
52612024-02-15T02:59:27.382Z 20 400-419 41 41 41 78 78 78 F F F
52622024-02-15T02:59:27.383Z 21 420-439 44 44 44 83 83 83 F T T <---
52632024-02-15T02:59:27.383Z 22 440-459 44 44 44 83 83 83 F F F
52642024-02-15T02:59:27.383Z 23 460-479 42 42 42 79 79 79 F F F
52652024-02-15T02:59:27.383Z 24 480-499 45 45 45 84 84 84 F F F
52662024-02-15T02:59:27.383Z 25 500-519 44 46 46 83 87 87 F F F <---
52672024-02-15T02:59:27.384Z 26 520-539 44 44 44 83 83 83 F F F
52682024-02-15T02:59:27.384Z 27 540-559 45 45 45 84 84 84 F F F
52692024-02-15T02:59:27.384Z 28 560-579 45 46 46 84 86 86 F F F <---
52702024-02-15T02:59:27.385Z 29 580-599 41 41 41 78 78 78 F F F
52712024-02-15T02:59:27.385ZMax gen: 46, Max flush: 87
52722024-02-15T02:59:27.385ZError: Difference in extent metadata found!
52732024-02-15T02:59:27.385ZA Difference in extent metadata is expected here
52742024-02-15T02:59:27.385ZOn loop 23
52752024-02-15T02:59:27.386Z
52762024-02-15T02:59:27.386ZVerifying data now
52772024-02-15T02:59:32.981ZNo change after verify
52782024-02-15T02:59:32.981ZLoop: 23 Downstairs dump after verify (and repair):
52792024-02-15T02:59:33.001ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
52802024-02-15T02:59:33.001Z 0 000-019 47 47 47 88 88 88 F F F
52812024-02-15T02:59:33.001Z 1 020-039 47 47 47 88 88 88 F F F
52822024-02-15T02:59:33.002Z 2 040-059 47 47 47 88 88 88 F F F
52832024-02-15T02:59:33.003Z 3 060-079 44 44 44 83 83 83 F F F
52842024-02-15T02:59:33.008Z 4 080-099 45 45 45 84 84 84 F F F
52852024-02-15T02:59:33.009Z 5 100-119 45 45 45 84 84 84 F F F
52862024-02-15T02:59:33.009Z 6 120-139 46 46 46 87 87 87 F F F
52872024-02-15T02:59:33.009Z 7 140-159 46 46 46 85 85 85 F F F
52882024-02-15T02:59:33.009Z 8 160-179 47 47 47 88 88 88 F F F
52892024-02-15T02:59:33.010Z 9 180-199 47 47 47 88 88 88 F F F
52902024-02-15T02:59:33.010Z 10 200-219 44 44 44 83 83 83 F F F
52912024-02-15T02:59:33.010Z 11 220-239 43 43 43 80 80 80 F F F
52922024-02-15T02:59:33.010Z 12 240-259 46 46 46 87 87 87 F F F
52932024-02-15T02:59:33.010Z 13 260-279 44 44 44 83 83 83 F F F
52942024-02-15T02:59:33.010Z 14 280-299 44 44 44 81 81 81 F F F
52952024-02-15T02:59:33.011Z 15 300-319 40 40 40 77 77 77 F F F
52962024-02-15T02:59:33.011Z 16 320-339 47 47 47 88 88 88 F F F
52972024-02-15T02:59:33.011Z 17 340-359 42 42 42 79 79 79 F F F
52982024-02-15T02:59:33.011Z 18 360-379 42 42 42 79 79 79 F F F
52992024-02-15T02:59:33.011Z 19 380-399 44 44 44 81 81 81 F F F
53002024-02-15T02:59:33.011Z 20 400-419 41 41 41 78 78 78 F F F
53012024-02-15T02:59:33.012Z 21 420-439 47 47 47 88 88 88 F F F
53022024-02-15T02:59:33.012Z 22 440-459 44 44 44 83 83 83 F F F
53032024-02-15T02:59:33.013Z 23 460-479 42 42 42 79 79 79 F F F
53042024-02-15T02:59:33.013Z 24 480-499 45 45 45 84 84 84 F F F
53052024-02-15T02:59:33.013Z 25 500-519 46 46 46 87 87 87 F F F
53062024-02-15T02:59:33.013Z 26 520-539 44 44 44 83 83 83 F F F
53072024-02-15T02:59:33.014Z 27 540-559 45 45 45 84 84 84 F F F
53082024-02-15T02:59:33.014Z 28 560-579 46 46 46 86 86 86 F F F
53092024-02-15T02:59:33.014Z 29 580-599 41 41 41 78 78 78 F F F
53102024-02-15T02:59:33.014ZMax gen: 47, Max flush: 88
53112024-02-15T02:59:33.014Z
53122024-02-15T02:59:33.014ZBegin loop 24
53132024-02-15T02:59:33.014ZDownstairs to restart: 1
53142024-02-15T02:59:33.039Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.95166236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327}
53152024-02-15T02:59:33.040Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.951933646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327}
53162024-02-15T02:59:33.040Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.951962922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327}
53172024-02-15T02:59:33.040ZCrucible runtime is spawned
53182024-02-15T02:59:33.041Z{"msg":"Crucible 50f9aae1-7ec8-4b69-906f-0e0c0f30cc48 has session id: f3b7c1be-6e61-496c-ba29-73c659b8b5fe","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.951999198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53192024-02-15T02:59:33.041Z{"msg":"Upstairs opts: Upstairs UUID: 50f9aae1-7ec8-4b69-906f-0e0c0f30cc48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952045072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53202024-02-15T02:59:33.041Z{"msg":"Crucible stats registered with UUID: 50f9aae1-7ec8-4b69-906f-0e0c0f30cc48","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952081017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53212024-02-15T02:59:33.041Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952098385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327}
53222024-02-15T02:59:33.042Z{"msg":"50f9aae1-7ec8-4b69-906f-0e0c0f30cc48 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952189614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53232024-02-15T02:59:33.042Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.9522214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53242024-02-15T02:59:33.042Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952259145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53252024-02-15T02:59:33.042Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.95229604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53262024-02-15T02:59:33.042Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952344264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53272024-02-15T02:59:33.042Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952372151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53282024-02-15T02:59:33.043Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952407247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53292024-02-15T02:59:33.043Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952434343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53302024-02-15T02:59:33.043Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952476278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53312024-02-15T02:59:33.043Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952610831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53322024-02-15T02:59:33.043Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952656105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53332024-02-15T02:59:33.043Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:31.952689151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53342024-02-15T02:59:33.043Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952731336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53352024-02-15T02:59:33.044Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:31.952767511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53362024-02-15T02:59:33.044Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:31.952798378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53372024-02-15T02:59:33.044Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952822035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53382024-02-15T02:59:33.044Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.9528544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53392024-02-15T02:59:33.045Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.952874938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53402024-02-15T02:59:33.045Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.953262349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53412024-02-15T02:59:33.045Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.953296245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53422024-02-15T02:59:33.045Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.953311713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53432024-02-15T02:59:33.046Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.953326031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53442024-02-15T02:59:33.046Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.953941214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53452024-02-15T02:59:33.046Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.953987129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53462024-02-15T02:59:33.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.954588673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53472024-02-15T02:59:33.046Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.954670723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53482024-02-15T02:59:33.046Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.954707028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53492024-02-15T02:59:33.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.954722656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53502024-02-15T02:59:33.046Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.954744394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53512024-02-15T02:59:33.046Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:31.954758312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53522024-02-15T02:59:34.021Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.954133863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53532024-02-15T02:59:34.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.954208294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53542024-02-15T02:59:34.035Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.955206709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53552024-02-15T02:59:34.035Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.955254103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53562024-02-15T02:59:34.035Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.955781367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53572024-02-15T02:59:34.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956385911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53582024-02-15T02:59:34.036Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956413668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53592024-02-15T02:59:34.036Z{"msg":"[0]R flush_numbers[0..12]: [88, 88, 88, 83, 84, 84, 87, 85, 88, 88, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956443854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53602024-02-15T02:59:34.036Z{"msg":"[0]R generation[0..12]: [47, 47, 47, 44, 45, 45, 46, 46, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956460042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53612024-02-15T02:59:34.036Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956487828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53622024-02-15T02:59:34.036Z{"msg":"[1]R flush_numbers[0..12]: [88, 88, 88, 83, 84, 84, 87, 85, 88, 88, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956511056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53632024-02-15T02:59:34.036Z{"msg":"[1]R generation[0..12]: [47, 47, 47, 44, 45, 45, 46, 46, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956525414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53642024-02-15T02:59:34.036Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956539852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53652024-02-15T02:59:34.036Z{"msg":"[2]R flush_numbers[0..12]: [88, 88, 88, 83, 84, 84, 87, 85, 88, 88, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956560819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53662024-02-15T02:59:34.036Z{"msg":"[2]R generation[0..12]: [47, 47, 47, 44, 45, 45, 46, 46, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956575108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53672024-02-15T02:59:34.036Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956589376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53682024-02-15T02:59:34.036Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956602474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53692024-02-15T02:59:34.037Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956621832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53702024-02-15T02:59:34.037ZWait for a query_work_queue command to finish before sending IO
53712024-02-15T02:59:34.037Z{"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956649528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53722024-02-15T02:59:34.037Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956664236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53732024-02-15T02:59:34.037Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956691213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53742024-02-15T02:59:34.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956706881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53752024-02-15T02:59:34.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956740907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53762024-02-15T02:59:34.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956775123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53772024-02-15T02:59:34.037Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956813618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53782024-02-15T02:59:34.037Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956841664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53792024-02-15T02:59:34.037Z{"msg":"50f9aae1-7ec8-4b69-906f-0e0c0f30cc48 is now active with session: f3b7c1be-6e61-496c-ba29-73c659b8b5fe","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956878819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53802024-02-15T02:59:34.037Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956902876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327,"session_id":"f3b7c1be-6e61-496c-ba29-73c659b8b5fe"}
53812024-02-15T02:59:34.037Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:32.956935143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327}
53822024-02-15T02:59:34.037ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
53832024-02-15T02:59:34.038ZLoading write count information from file "/tmp/test_repair_verify.data"
53842024-02-15T02:59:34.038ZRun Repair workload
53852024-02-15T02:59:34.038Z01/30 Write block 509 len 24576 data: 5 4 5 7 7 7
53862024-02-15T02:59:34.038Z02/30 Read block 353 len 28672
53872024-02-15T02:59:34.038Z03/30 Write block 489 len 4096 data: 6
53882024-02-15T02:59:34.038Z04/30 Read block 300 len 12288
53892024-02-15T02:59:34.038Z05/30 Write block 471 len 32768 data: 7 6 5 4 2 2 3 4
53902024-02-15T02:59:34.038Z06/30 Write block 46 len 16384 data: 6 6 5 4
53912024-02-15T02:59:34.038Z07/30 Write block 440 len 28672 data: 6 7 7 8 7 8 8
53922024-02-15T02:59:34.038Z08/30 Write block 394 len 8192 data: 5 6
53932024-02-15T02:59:34.038Z09/30 Read block 480 len 24576
53942024-02-15T02:59:34.045Z10/30 Read block 245 len 28672
53952024-02-15T02:59:34.045Z11/30 Write block 334 len 24576 data: 7 8 7 6 6 4
53962024-02-15T02:59:34.045Z12/30 Read block 228 len 8192
53972024-02-15T02:59:34.045Z13/30 Read block 298 len 28672
53982024-02-15T02:59:34.046Z14/30 Read block 162 len 4096
53992024-02-15T02:59:34.046Z15/30 Write block 508 len 40960 data: 4 6 5 6 8 8 8 6 6 4
54002024-02-15T02:59:34.046Z16/30 Flush
54012024-02-15T02:59:34.046Z17/30 Write block 301 len 12288 data: 8 8 9
54022024-02-15T02:59:34.046Z18/30 Read block 20 len 32768
54032024-02-15T02:59:34.046Z19/30 Read block 457 len 24576
54042024-02-15T02:59:34.046Z20/30 Read block 570 len 28672
54052024-02-15T02:59:34.046Z21/30 Read block 588 len 36864
54062024-02-15T02:59:34.046Z22/30 Read block 413 len 36864
54072024-02-15T02:59:34.046Z23/30 Write block 24 len 4096 data: 2
54082024-02-15T02:59:34.046Z24/30 Write block 545 len 8192 data: 3 3
54092024-02-15T02:59:34.046Z25/30 Read block 586 len 32768
54102024-02-15T02:59:34.477Z26/30 Write block 582 len 40960 data: 4 3 3 3 4 3 3 6 5 5
54112024-02-15T02:59:34.477Z27/30 Read block 143 len 28672
54122024-02-15T02:59:34.477Z28/30 Write block 299 len 28672 data: 6 6 9 9 10 10 11
54132024-02-15T02:59:34.478Z29/30 Write block 570 len 12288 data: 3 4 3
54142024-02-15T02:59:34.479Z30/30 Write block 4 len 32768 data: 5 5 5 5 6 6 7 4
54152024-02-15T02:59:34.479Z----------------------------------------------------------------
54162024-02-15T02:59:34.479Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
54172024-02-15T02:59:34.479ZStates: Active Active Active
54182024-02-15T02:59:34.479ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54192024-02-15T02:59:34.479Z 1 Acked 1000 Write 6 Done Done Done false
54202024-02-15T02:59:34.479Z 2 Acked 1001 Read 7 Done Done Done false
54212024-02-15T02:59:34.479Z 3 Acked 1002 Write 1 Done Done Done false
54222024-02-15T02:59:34.480Z 4 Acked 1003 Read 3 Done Done Done false
54232024-02-15T02:59:34.480Z 5 Acked 1004 Write 8 Done Done Done false
54242024-02-15T02:59:34.480Z 6 Acked 1005 Write 4 Done Done Done false
54252024-02-15T02:59:34.480Z 7 Acked 1006 Write 7 Done Done Done false
54262024-02-15T02:59:34.480Z 8 Acked 1007 Write 2 Done Done Done false
54272024-02-15T02:59:34.480Z 9 Acked 1008 Read 6 Done Done Done false
54282024-02-15T02:59:34.480Z 10 Acked 1009 Read 7 Done Done Done false
54292024-02-15T02:59:34.481Z 11 Acked 1010 Write 6 Done Done Done false
54302024-02-15T02:59:34.481Z 12 Acked 1011 Read 2 Done Done Done false
54312024-02-15T02:59:34.481Z 13 Acked 1012 Read 7 Done Sent Done false
54322024-02-15T02:59:34.481Z 14 Acked 1013 Read 1 Done Sent Done false
54332024-02-15T02:59:34.481Z 15 Acked 1014 Write 10 Done Sent Done false
54342024-02-15T02:59:34.481Z 16 Acked 1015 Flush 0 Done Sent Done false
54352024-02-15T02:59:34.481Z 17 Acked 1016 Write 3 Done Sent Done false
54362024-02-15T02:59:34.481Z 18 Acked 1017 Read 8 Done Sent Done false
54372024-02-15T02:59:34.482Z 19 Acked 1018 Read 6 Done Sent Done false
54382024-02-15T02:59:34.482Z 20 Acked 1019 Read 7 Done Sent Done false
54392024-02-15T02:59:34.482Z 21 Acked 1020 Read 9 Done Sent Done false
54402024-02-15T02:59:34.482Z 22 Acked 1021 Read 9 Done Sent Done false
54412024-02-15T02:59:34.483Z 23 Acked 1022 Write 1 Done Sent Done false
54422024-02-15T02:59:34.483Z 24 Acked 1023 Write 2 Done Sent Done false
54432024-02-15T02:59:34.483Z 25 Acked 1024 Read 8 Done Sent Done false
54442024-02-15T02:59:34.483Z 26 Acked 1025 Write 10 Done Sent Done false
54452024-02-15T02:59:34.483Z 27 Acked 1026 Read 7 Done Sent Done false
54462024-02-15T02:59:34.483Z 28 Acked 1027 Write 7 Done Sent Done false
54472024-02-15T02:59:34.483Z 29 Acked 1028 Write 3 Done Sent Done false
54482024-02-15T02:59:34.484Z 30 Acked 1029 Write 8 Sent Sent Sent false
54492024-02-15T02:59:34.484Z STATES DS:0 DS:1 DS:2 TOTAL
54502024-02-15T02:59:34.484Z New 0 0 0 0
54512024-02-15T02:59:34.484Z Sent 1 18 1 20
54522024-02-15T02:59:34.484Z Done 29 12 29 70
54532024-02-15T02:59:34.484Z Skipped 0 0 0 0
54542024-02-15T02:59:34.484Z Error 0 0 0 0
54552024-02-15T02:59:34.484ZLast Flush: 1015 0 1015
54562024-02-15T02:59:34.485ZDownstairs last five completed:
54572024-02-15T02:59:34.485ZUpstairs last five completed: 30 29 28 27 26
54582024-02-15T02:59:34.485Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:32.98688703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1327}
54592024-02-15T02:59:34.485ZWrote out file "/tmp/test_repair_verify.data"
54602024-02-15T02:59:34.505Z
54612024-02-15T02:59:34.505ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
54622024-02-15T02:59:34.505ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
54632024-02-15T02:59:34.506Z 0 000-019 47 47 47 88 88 88 T F T <---
54642024-02-15T02:59:34.506Z 1 020-039 47 47 47 88 88 88 T F T <---
54652024-02-15T02:59:34.506Z 2 040-059 48 47 48 89 88 89 F T F <---
54662024-02-15T02:59:34.506Z 3 060-079 44 44 44 83 83 83 F F F
54672024-02-15T02:59:34.506Z 4 080-099 45 45 45 84 84 84 F F F
54682024-02-15T02:59:34.506Z 5 100-119 45 45 45 84 84 84 F F F
54692024-02-15T02:59:34.506Z 6 120-139 46 46 46 87 87 87 F F F
54702024-02-15T02:59:34.507Z 7 140-159 46 46 46 85 85 85 F F F
54712024-02-15T02:59:34.507Z 8 160-179 47 47 47 88 88 88 F F F
54722024-02-15T02:59:34.507Z 9 180-199 47 47 47 88 88 88 F F F
54732024-02-15T02:59:34.507Z 10 200-219 44 44 44 83 83 83 F F F
54742024-02-15T02:59:34.507Z 11 220-239 43 43 43 80 80 80 F F F
54752024-02-15T02:59:34.507Z 12 240-259 46 46 46 87 87 87 F F F
54762024-02-15T02:59:34.507Z 13 260-279 44 44 44 83 83 83 F F F
54772024-02-15T02:59:34.507Z 14 280-299 44 44 44 81 81 81 T F T <---
54782024-02-15T02:59:34.507Z 15 300-319 40 40 40 77 77 77 T F T <---
54792024-02-15T02:59:34.507Z 16 320-339 48 47 48 89 88 89 F T F <---
54802024-02-15T02:59:34.508Z 17 340-359 42 42 42 79 79 79 F F F
54812024-02-15T02:59:34.508Z 18 360-379 42 42 42 79 79 79 F F F
54822024-02-15T02:59:34.508Z 19 380-399 48 44 48 89 81 89 F T F <---
54832024-02-15T02:59:34.508Z 20 400-419 41 41 41 78 78 78 F F F
54842024-02-15T02:59:34.508Z 21 420-439 47 47 47 88 88 88 F F F
54852024-02-15T02:59:34.509Z 22 440-459 48 44 48 89 83 89 F T F <---
54862024-02-15T02:59:34.509Z 23 460-479 48 42 48 89 79 89 F T F <---
54872024-02-15T02:59:34.509Z 24 480-499 48 45 48 89 84 89 F T F <---
54882024-02-15T02:59:34.509Z 25 500-519 48 46 48 89 87 89 F T F <---
54892024-02-15T02:59:34.509Z 26 520-539 44 44 44 83 83 83 F F F
54902024-02-15T02:59:34.510Z 27 540-559 45 45 45 84 84 84 T F T <---
54912024-02-15T02:59:34.510Z 28 560-579 46 46 46 86 86 86 T F T <---
54922024-02-15T02:59:34.510Z 29 580-599 41 41 41 78 78 78 T F T <---
54932024-02-15T02:59:34.510ZMax gen: 48, Max flush: 89
54942024-02-15T02:59:34.511ZError: Difference in extent metadata found!
54952024-02-15T02:59:34.511ZA Difference in extent metadata is expected here
54962024-02-15T02:59:34.511ZOn loop 24
54972024-02-15T02:59:34.511Z
54982024-02-15T02:59:34.511ZVerifying data now
54992024-02-15T02:59:39.807ZNo change after verify
55002024-02-15T02:59:39.807ZLoop: 24 Downstairs dump after verify (and repair):
55012024-02-15T02:59:40.238ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55022024-02-15T02:59:40.238Z 0 000-019 49 49 49 90 90 90 F F F
55032024-02-15T02:59:40.239Z 1 020-039 49 49 49 90 90 90 F F F
55042024-02-15T02:59:40.239Z 2 040-059 48 48 48 89 89 89 F F F
55052024-02-15T02:59:40.239Z 3 060-079 44 44 44 83 83 83 F F F
55062024-02-15T02:59:40.239Z 4 080-099 45 45 45 84 84 84 F F F
55072024-02-15T02:59:40.239Z 5 100-119 45 45 45 84 84 84 F F F
55082024-02-15T02:59:40.240Z 6 120-139 46 46 46 87 87 87 F F F
55092024-02-15T02:59:40.240Z 7 140-159 46 46 46 85 85 85 F F F
55102024-02-15T02:59:40.240Z 8 160-179 47 47 47 88 88 88 F F F
55112024-02-15T02:59:40.240Z 9 180-199 47 47 47 88 88 88 F F F
55122024-02-15T02:59:40.240Z 10 200-219 44 44 44 83 83 83 F F F
55132024-02-15T02:59:40.240Z 11 220-239 43 43 43 80 80 80 F F F
55142024-02-15T02:59:40.241Z 12 240-259 46 46 46 87 87 87 F F F
55152024-02-15T02:59:40.241Z 13 260-279 44 44 44 83 83 83 F F F
55162024-02-15T02:59:40.241Z 14 280-299 49 49 49 90 90 90 F F F
55172024-02-15T02:59:40.241Z 15 300-319 49 49 49 90 90 90 F F F
55182024-02-15T02:59:40.241Z 16 320-339 48 48 48 89 89 89 F F F
55192024-02-15T02:59:40.242Z 17 340-359 42 42 42 79 79 79 F F F
55202024-02-15T02:59:40.242Z 18 360-379 42 42 42 79 79 79 F F F
55212024-02-15T02:59:40.242Z 19 380-399 48 48 48 89 89 89 F F F
55222024-02-15T02:59:40.242Z 20 400-419 41 41 41 78 78 78 F F F
55232024-02-15T02:59:40.242Z 21 420-439 47 47 47 88 88 88 F F F
55242024-02-15T02:59:40.243Z 22 440-459 48 48 48 89 89 89 F F F
55252024-02-15T02:59:40.243Z 23 460-479 48 48 48 89 89 89 F F F
55262024-02-15T02:59:40.243Z 24 480-499 48 48 48 89 89 89 F F F
55272024-02-15T02:59:40.243Z 25 500-519 48 48 48 89 89 89 F F F
55282024-02-15T02:59:40.243Z 26 520-539 44 44 44 83 83 83 F F F
55292024-02-15T02:59:40.243Z 27 540-559 49 49 49 90 90 90 F F F
55302024-02-15T02:59:40.244Z 28 560-579 49 49 49 90 90 90 F F F
55312024-02-15T02:59:40.244Z 29 580-599 49 49 49 90 90 90 F F F
55322024-02-15T02:59:40.244ZMax gen: 49, Max flush: 90
55332024-02-15T02:59:40.245Z
55342024-02-15T02:59:40.246ZBegin loop 25
55352024-02-15T02:59:40.246ZDownstairs to restart: 1
55362024-02-15T02:59:40.278Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.764793235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335}
55372024-02-15T02:59:40.278Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765080879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335}
55382024-02-15T02:59:40.279Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765107536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335}
55392024-02-15T02:59:40.279Z{"msg":"Crucible fec6a062-242f-4a3d-87e3-c484d3fbc3de has session id: a08e8652-9586-48c0-a59f-4908c1279430","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765135962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55402024-02-15T02:59:40.279Z{"msg":"Upstairs opts: Upstairs UUID: fec6a062-242f-4a3d-87e3-c484d3fbc3de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765201974Z"Crucible runtime is spawned
55412024-02-15T02:59:40.279Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55422024-02-15T02:59:40.279Z{"msg":"Crucible stats registered with UUID: fec6a062-242f-4a3d-87e3-c484d3fbc3de","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765252927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55432024-02-15T02:59:40.280Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765331468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335}
55442024-02-15T02:59:40.280Z{"msg":"fec6a062-242f-4a3d-87e3-c484d3fbc3de active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765410198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55452024-02-15T02:59:40.281Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765449183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55462024-02-15T02:59:40.282Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765486408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55472024-02-15T02:59:40.282Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765515375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55482024-02-15T02:59:40.282Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765542541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55492024-02-15T02:59:40.282Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765578557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55502024-02-15T02:59:40.282Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765608283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55512024-02-15T02:59:40.283Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765684324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55522024-02-15T02:59:40.283Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765716609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55532024-02-15T02:59:40.283Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765739357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55542024-02-15T02:59:40.283Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765763083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55552024-02-15T02:59:40.283Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765787691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55562024-02-15T02:59:40.284Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:38.765814177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55572024-02-15T02:59:40.284Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:38.765844104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55582024-02-15T02:59:40.284Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:38.765896797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55592024-02-15T02:59:40.284Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765932772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55602024-02-15T02:59:40.284Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.765948071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55612024-02-15T02:59:40.284Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.766038669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55622024-02-15T02:59:40.284Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.766490103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55632024-02-15T02:59:40.284Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.766533177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55642024-02-15T02:59:40.284Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.766558464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55652024-02-15T02:59:40.285Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.766598009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55662024-02-15T02:59:40.285Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.76706852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55672024-02-15T02:59:40.285Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.767215202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55682024-02-15T02:59:40.285Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.767639548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55692024-02-15T02:59:40.285Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.767719829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55702024-02-15T02:59:40.285Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.767774092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55712024-02-15T02:59:40.285Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.767828335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55722024-02-15T02:59:40.285Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.767860281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55732024-02-15T02:59:40.285Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:38.767903835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55742024-02-15T02:59:41.575Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.767540424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55752024-02-15T02:59:41.576Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.767799991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55762024-02-15T02:59:41.576Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.768806575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55772024-02-15T02:59:41.576Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.768855709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55782024-02-15T02:59:41.576Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.769382603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55792024-02-15T02:59:41.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770026552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55802024-02-15T02:59:41.576Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770078206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55812024-02-15T02:59:41.576Z{"msg":"[0]R flush_numbers[0..12]: [90, 90, 89, 83, 84, 84, 87, 85, 88, 88, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770099313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55822024-02-15T02:59:41.576Z{"msg":"[0]R generation[0..12]: [49, 49, 48, 44, 45, 45, 46, 46, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770127359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55832024-02-15T02:59:41.576Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770146967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55842024-02-15T02:59:41.577Z{"msg":"[1]R flush_numbers[0..12]: [90, 90, 89, 83, 84, 84, 87, 85, 88, 88, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770174364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55852024-02-15T02:59:41.577Z{"msg":"[1]R generation[0..12]: [49, 49, 48, 44, 45, 45, 46, 46, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770192851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55862024-02-15T02:59:41.577Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770219268Z","hostname":"Wait for a query_work_queue command to finish before sending IO
55872024-02-15T02:59:41.577Zip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55882024-02-15T02:59:41.577Z{"msg":"[2]R flush_numbers[0..12]: [90, 90, 89, 83, 84, 84, 87, 85, 88, 88, 83, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770246265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55892024-02-15T02:59:41.577Z{"msg":"[2]R generation[0..12]: [49, 49, 48, 44, 45, 45, 46, 46, 47, 47, 44, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770273221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55902024-02-15T02:59:41.578Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770289139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55912024-02-15T02:59:41.578Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770309167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55922024-02-15T02:59:41.578Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770323155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55932024-02-15T02:59:41.578Z{"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770335964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55942024-02-15T02:59:41.578Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770355851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55952024-02-15T02:59:41.578Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770369729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55962024-02-15T02:59:41.578Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770381878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55972024-02-15T02:59:41.579Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770401985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55982024-02-15T02:59:41.579Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770415414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
55992024-02-15T02:59:41.579Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770436371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
56002024-02-15T02:59:41.579Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770449799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
56012024-02-15T02:59:41.579Z{"msg":"fec6a062-242f-4a3d-87e3-c484d3fbc3de is now active with session: a08e8652-9586-48c0-a59f-4908c1279430","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770463128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
56022024-02-15T02:59:41.579Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770482455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335,"session_id":"a08e8652-9586-48c0-a59f-4908c1279430"}
56032024-02-15T02:59:41.579Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:39.770494893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335}
56042024-02-15T02:59:41.580ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56052024-02-15T02:59:41.580ZLoading write count information from file "/tmp/test_repair_verify.data"
56062024-02-15T02:59:41.580ZRun Repair workload
56072024-02-15T02:59:41.580Z01/30 Write block 498 len 32768 data: 4 4 7 8 10 9 8 7
56082024-02-15T02:59:41.580Z02/30 Write block 495 len 24576 data: 5 5 5 5 5 8
56092024-02-15T02:59:41.580Z03/30 Read block 294 len 40960
56102024-02-15T02:59:41.580Z04/30 Read block 580 len 16384
56112024-02-15T02:59:41.580Z05/30 Write block 162 len 20480 data: 5 5 3 4 4
56122024-02-15T02:59:41.580Z06/30 Flush
56132024-02-15T02:59:41.580Z07/30 Write block 490 len 40960 data: 7 7 5 5 6 6 6 6 6 6
56142024-02-15T02:59:41.580Z08/30 Read block 292 len 16384
56152024-02-15T02:59:41.580Z09/30 Read block 149 len 24576
56162024-02-15T02:59:41.581Z10/30 Write block 353 len 8192 data: 5 4
56172024-02-15T02:59:41.581Z11/30 Read block 447 len 28672
56182024-02-15T02:59:41.581Z12/30 Read block 360 len 24576
56192024-02-15T02:59:41.581Z13/30 Read block 395 len 8192
56202024-02-15T02:59:41.581Z14/30 Read block 464 len 12288
56212024-02-15T02:59:41.581Z15/30 Write block 370 len 28672 data: 4 4 4 4 4 2 4
56222024-02-15T02:59:41.581Z16/30 Write block 441 len 20480 data: 8 8 9 8 9
56232024-02-15T02:59:41.581Z17/30 Flush
56242024-02-15T02:59:41.581Z18/30 Write block 156 len 16384 data: 4 5 4 4
56252024-02-15T02:59:41.581Z19/30 Read block 308 len 32768
56262024-02-15T02:59:41.582Z20/30 Write block 559 len 16384 data: 7 6 5 5
56272024-02-15T02:59:41.582Z21/30 Write block 225 len 8192 data: 6 8
56282024-02-15T02:59:41.582Z22/30 Read block 479 len 24576
56292024-02-15T02:59:41.582Z23/30 Read block 202 len 4096
56302024-02-15T02:59:41.582Z24/30 Read block 474 len 8192
56312024-02-15T02:59:41.582Z25/30 Flush
56322024-02-15T02:59:41.582Z26/30 Write block 360 len 28672 data: 4 4 5 4 5 5 6
56332024-02-15T02:59:41.582Z27/30 Write block 117 len 32768 data: 6 5 5 5 6 5 5 4
56342024-02-15T02:59:41.582Z28/30 Write block 189 len 36864 data: 6 6 6 7 9 8 8 6 5
56352024-02-15T02:59:41.583Z29/30 Read block 146 len 32768
56362024-02-15T02:59:41.583Z30/30 Write block 183 len 36864 data: 4 5 6 6 4 4 7 7 7
56372024-02-15T02:59:41.583Z----------------------------------------------------------------
56382024-02-15T02:59:41.583Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
56392024-02-15T02:59:41.583ZStates: Active Active Active
56402024-02-15T02:59:41.583ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56412024-02-15T02:59:41.583Z 1 Acked 1000 Write 8 Done Done Done false
56422024-02-15T02:59:41.583Z 2 Acked 1001 Write 6 Done Sent Done false
56432024-02-15T02:59:41.583Z 3 Acked 1002 Read 10 Done Sent Done false
56442024-02-15T02:59:41.583Z 4 Acked 1003 Read 4 Done Sent Done false
56452024-02-15T02:59:41.583Z 5 Acked 1004 Write 5 Done Sent Done false
56462024-02-15T02:59:41.584Z 6 Acked 1005 Flush 0 Done Sent Done false
56472024-02-15T02:59:41.584Z 7 Acked 1006 Write 10 Done Sent Done false
56482024-02-15T02:59:41.584Z 8 Acked 1007 Read 4 Done Sent Done false
56492024-02-15T02:59:41.584Z 9 Acked 1008 Read 6 Done Sent Done false
56502024-02-15T02:59:41.584Z 10 Acked 1009 Write 2 Done Sent Done false
56512024-02-15T02:59:41.585Z 11 Acked 1010 Read 7 Done Sent Done false
56522024-02-15T02:59:41.585Z 12 Acked 1011 Read 6 Done Sent Done false
56532024-02-15T02:59:41.585Z 13 Acked 1012 Read 2 Done Sent Done false
56542024-02-15T02:59:41.585Z 14 Acked 1013 Read 3 Done Sent Done false
56552024-02-15T02:59:41.585Z 15 Acked 1014 Write 7 Done Sent Done false
56562024-02-15T02:59:41.586Z 16 Acked 1015 Write 5 Done Sent Done false
56572024-02-15T02:59:41.586Z 17 Acked 1016 Flush 0 Done Sent Done false
56582024-02-15T02:59:41.586Z 18 Acked 1017 Write 4 Done Sent Done false
56592024-02-15T02:59:41.586Z 19 Acked 1018 Read 8 Done Sent Done false
56602024-02-15T02:59:41.586Z 20 Acked 1019 Write 4 Done Sent Done false
56612024-02-15T02:59:41.586Z 21 Acked 1020 Write 2 Done Sent Done false
56622024-02-15T02:59:41.586Z 22 Acked 1021 Read 6 Done Sent Done false
56632024-02-15T02:59:41.586Z 23 Acked 1022 Read 1 Done Sent Done false
56642024-02-15T02:59:41.586Z 24 Acked 1023 Read 2 Done Sent Done false
56652024-02-15T02:59:41.587Z 25 Acked 1024 Flush 0 Done Sent Done false
56662024-02-15T02:59:41.587Z 26 Acked 1025 Write 7 Done Sent Done false
56672024-02-15T02:59:41.587Z 27 Acked 1026 Write 8 Done Sent Done false
56682024-02-15T02:59:41.587Z 28 Acked 1027 Write 9 Done Sent Done false
56692024-02-15T02:59:41.587Z 29 Acked 1028 Read 8 Done Sent Done false
56702024-02-15T02:59:41.587Z 30 Acked 1029 Write 9 Sent Sent Sent false
56712024-02-15T02:59:41.587Z STATES DS:0 DS:1 DS:2 TOTAL
56722024-02-15T02:59:41.588Z New 0 0 0 0
56732024-02-15T02:59:41.588Z Sent 1 29 1 31
56742024-02-15T02:59:41.618Z Done 29 1 29 59
56752024-02-15T02:59:41.619Z Skipped 0 0 0 0
56762024-02-15T02:59:41.619Z Error 0 0 0 0
56772024-02-15T02:59:41.620ZLast Flush: 1024 0 1024
56782024-02-15T02:59:41.620ZDownstairs last five completed:
56792024-02-15T02:59:41.620ZUpstairs last five completed: 30 29 28 27 26
56802024-02-15T02:59:41.620Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:39.803310722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1335}
56812024-02-15T02:59:41.620ZWrote out file "/tmp/test_repair_verify.data"
56822024-02-15T02:59:41.621Z
56832024-02-15T02:59:41.621ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
56842024-02-15T02:59:41.621ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56852024-02-15T02:59:41.621Z 0 000-019 49 49 49 90 90 90 F F F
56862024-02-15T02:59:41.621Z 1 020-039 49 49 49 90 90 90 F F F
56872024-02-15T02:59:41.621Z 2 040-059 48 48 48 89 89 89 F F F
56882024-02-15T02:59:41.622Z 3 060-079 44 44 44 83 83 83 F F F
56892024-02-15T02:59:41.622Z 4 080-099 45 45 45 84 84 84 F F F
56902024-02-15T02:59:41.622Z 5 100-119 45 45 45 84 84 84 T F T <---
56912024-02-15T02:59:41.622Z 6 120-139 46 46 46 87 87 87 T F T <---
56922024-02-15T02:59:41.622Z 7 140-159 50 46 50 93 85 93 F F F <---
56932024-02-15T02:59:41.623Z 8 160-179 50 47 50 91 88 91 F F F <---
56942024-02-15T02:59:41.623Z 9 180-199 47 47 47 88 88 88 T F T <---
56952024-02-15T02:59:41.623Z 10 200-219 44 44 44 83 83 83 F F F
56962024-02-15T02:59:41.623Z 11 220-239 50 43 50 93 80 93 F F F <---
56972024-02-15T02:59:41.623Z 12 240-259 46 46 46 87 87 87 F F F
56982024-02-15T02:59:41.623Z 13 260-279 44 44 44 83 83 83 F F F
56992024-02-15T02:59:41.623Z 14 280-299 49 49 49 90 90 90 F F F
57002024-02-15T02:59:41.624Z 15 300-319 49 49 49 90 90 90 F F F
57012024-02-15T02:59:41.624Z 16 320-339 48 48 48 89 89 89 F F F
57022024-02-15T02:59:41.624Z 17 340-359 50 42 50 92 79 92 F F F <---
57032024-02-15T02:59:41.624Z 18 360-379 50 42 50 92 79 92 T F T <---
57042024-02-15T02:59:41.625Z 19 380-399 48 48 48 89 89 89 F F F
57052024-02-15T02:59:41.625Z 20 400-419 41 41 41 78 78 78 F F F
57062024-02-15T02:59:41.625Z 21 420-439 47 47 47 88 88 88 F F F
57072024-02-15T02:59:41.625Z 22 440-459 50 48 50 92 89 92 F F F <---
57082024-02-15T02:59:41.625Z 23 460-479 48 48 48 89 89 89 F F F
57092024-02-15T02:59:41.625Z 24 480-499 50 48 50 92 89 92 F T F <---
57102024-02-15T02:59:41.625Z 25 500-519 50 48 50 91 89 91 F T F <---
57112024-02-15T02:59:41.625Z 26 520-539 44 44 44 83 83 83 F F F
57122024-02-15T02:59:41.626Z 27 540-559 50 49 50 93 90 93 F F F <---
57132024-02-15T02:59:41.626Z 28 560-579 50 49 50 93 90 93 F F F <---
57142024-02-15T02:59:41.626Z 29 580-599 49 49 49 90 90 90 F F F
57152024-02-15T02:59:41.626ZMax gen: 50, Max flush: 93
57162024-02-15T02:59:41.626ZError: Difference in extent metadata found!
57172024-02-15T02:59:41.626ZA Difference in extent metadata is expected here
57182024-02-15T02:59:41.626ZOn loop 25
57192024-02-15T02:59:41.627Z
57202024-02-15T02:59:41.627ZVerifying data now
57212024-02-15T02:59:46.536ZNo change after verify
57222024-02-15T02:59:46.536ZLoop: 25 Downstairs dump after verify (and repair):
57232024-02-15T02:59:46.559ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57242024-02-15T02:59:46.559Z 0 000-019 49 49 49 90 90 90 F F F
57252024-02-15T02:59:46.560Z 1 020-039 49 49 49 90 90 90 F F F
57262024-02-15T02:59:46.560Z 2 040-059 48 48 48 89 89 89 F F F
57272024-02-15T02:59:46.560Z 3 060-079 44 44 44 83 83 83 F F F
57282024-02-15T02:59:46.560Z 4 080-099 45 45 45 84 84 84 F F F
57292024-02-15T02:59:46.560Z 5 100-119 51 51 51 94 94 94 F F F
57302024-02-15T02:59:46.560Z 6 120-139 51 51 51 94 94 94 F F F
57312024-02-15T02:59:46.560Z 7 140-159 50 50 50 93 93 93 F F F
57322024-02-15T02:59:46.560Z 8 160-179 50 50 50 91 91 91 F F F
57332024-02-15T02:59:46.560Z 9 180-199 51 51 51 94 94 94 F F F
57342024-02-15T02:59:46.560Z 10 200-219 44 44 44 83 83 83 F F F
57352024-02-15T02:59:46.560Z 11 220-239 50 50 50 93 93 93 F F F
57362024-02-15T02:59:46.560Z 12 240-259 46 46 46 87 87 87 F F F
57372024-02-15T02:59:46.561Z 13 260-279 44 44 44 83 83 83 F F F
57382024-02-15T02:59:46.561Z 14 280-299 49 49 49 90 90 90 F F F
57392024-02-15T02:59:46.561Z 15 300-319 49 49 49 90 90 90 F F F
57402024-02-15T02:59:46.561Z 16 320-339 48 48 48 89 89 89 F F F
57412024-02-15T02:59:46.561Z 17 340-359 50 50 50 92 92 92 F F F
57422024-02-15T02:59:46.561Z 18 360-379 51 51 51 94 94 94 F F F
57432024-02-15T02:59:46.561Z 19 380-399 48 48 48 89 89 89 F F F
57442024-02-15T02:59:46.561Z 20 400-419 41 41 41 78 78 78 F F F
57452024-02-15T02:59:46.561Z 21 420-439 47 47 47 88 88 88 F F F
57462024-02-15T02:59:46.561Z 22 440-459 50 50 50 92 92 92 F F F
57472024-02-15T02:59:46.561Z 23 460-479 48 48 48 89 89 89 F F F
57482024-02-15T02:59:46.561Z 24 480-499 50 50 50 92 92 92 F F F
57492024-02-15T02:59:46.562Z 25 500-519 50 50 50 91 91 91 F F F
57502024-02-15T02:59:46.562Z 26 520-539 44 44 44 83 83 83 F F F
57512024-02-15T02:59:46.562Z 27 540-559 50 50 50 93 93 93 F F F
57522024-02-15T02:59:46.562Z 28 560-579 50 50 50 93 93 93 F F F
57532024-02-15T02:59:46.562Z 29 580-599 49 49 49 90 90 90 F F F
57542024-02-15T02:59:46.562ZMax gen: 51, Max flush: 94
57552024-02-15T02:59:46.562Z
57562024-02-15T02:59:46.562ZBegin loop 26
57572024-02-15T02:59:46.562ZDownstairs to restart: 2
57582024-02-15T02:59:46.580Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.515850178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343}
57592024-02-15T02:59:46.580Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516199934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343}
57602024-02-15T02:59:46.619Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516241949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343}
57612024-02-15T02:59:46.619Z{"msg":"Crucible 25aefeed-d4ac-4c0e-a840-3dbc7e5249df has session id: 6f925db0-95cd-4580-b7d8-9e8b8115191d","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516297812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57622024-02-15T02:59:46.619ZCrucible runtime is spawned
57632024-02-15T02:59:46.619Z{"msg":"Upstairs opts: Upstairs UUID: 25aefeed-d4ac-4c0e-a840-3dbc7e5249df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516443024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57642024-02-15T02:59:46.619Z{"msg":"Crucible stats registered with UUID: 25aefeed-d4ac-4c0e-a840-3dbc7e5249df","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516498187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57652024-02-15T02:59:46.619Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516530913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343}
57662024-02-15T02:59:46.619Z{"msg":"25aefeed-d4ac-4c0e-a840-3dbc7e5249df active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.51663303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57672024-02-15T02:59:46.619Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516667376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57682024-02-15T02:59:46.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516693212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57692024-02-15T02:59:46.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516721769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57702024-02-15T02:59:46.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516754125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57712024-02-15T02:59:46.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516782471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57722024-02-15T02:59:46.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516807378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57732024-02-15T02:59:46.620Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516847133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57742024-02-15T02:59:46.620Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516924694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57752024-02-15T02:59:46.620Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.516969328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57762024-02-15T02:59:46.620Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.51710912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57772024-02-15T02:59:46.620Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:45.517151665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57782024-02-15T02:59:46.620Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:45.517178512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57792024-02-15T02:59:46.620Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.517203608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57802024-02-15T02:59:46.620Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:45.517245683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57812024-02-15T02:59:46.620Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.51727573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57822024-02-15T02:59:46.620Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.517291608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57832024-02-15T02:59:46.620Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.51735155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57842024-02-15T02:59:46.620Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.517674549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57852024-02-15T02:59:46.620Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.517714484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57862024-02-15T02:59:46.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.517739781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57872024-02-15T02:59:46.621Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.517772947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57882024-02-15T02:59:46.621Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518246378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57892024-02-15T02:59:46.621Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518357524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57902024-02-15T02:59:46.621Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518765943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57912024-02-15T02:59:46.621Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518808018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57922024-02-15T02:59:46.621Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518823276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57932024-02-15T02:59:46.621Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518875419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57942024-02-15T02:59:46.621Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518912125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57952024-02-15T02:59:46.621Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:45.518928692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57962024-02-15T02:59:47.585Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.518670338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57972024-02-15T02:59:47.585Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.518762626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57982024-02-15T02:59:47.611Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.519667603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
57992024-02-15T02:59:47.611Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.519714437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58002024-02-15T02:59:47.612Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.520191067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58012024-02-15T02:59:47.612Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.52080564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58022024-02-15T02:59:47.612Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.520862673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58032024-02-15T02:59:47.612Z{"msg":"[0]R flush_numbers[0..12]: [90, 90, 89, 83, 84, 94, 94, 93, 91, 94, 83, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.520907108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58042024-02-15T02:59:47.612Z{"msg":"[0]R generation[0..12]: [49, 49, 48, 44, 45, 51, 51, 50, 50, 51, 44, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.520951112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58052024-02-15T02:59:47.612Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.520997066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58062024-02-15T02:59:47.613Z{"msg":"[1]R flush_numbers[0..12]: [90, 90, 89, 83, 84, 94, 94, 93, 91, 94, 83, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521040951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58072024-02-15T02:59:47.613Z{"msg":"[1]R generation[0..12]: [49, 49, 48, 44, 45, 51, 51, 50, 50, 51, 44, 50]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
58082024-02-15T02:59:47.613Z,"time":"2024-02-15T02:59:46.521085675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58092024-02-15T02:59:47.613Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521139718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58102024-02-15T02:59:47.614Z{"msg":"[2]R flush_numbers[0..12]: [90, 90, 89, 83, 84, 94, 94, 93, 91, 94, 83, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521189722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58112024-02-15T02:59:47.614Z{"msg":"[2]R generation[0..12]: [49, 49, 48, 44, 45, 51, 51, 50, 50, 51, 44, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521239266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58122024-02-15T02:59:47.614Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.52128538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58132024-02-15T02:59:47.614Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521320076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58142024-02-15T02:59:47.614Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.52136481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58152024-02-15T02:59:47.614Z{"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521409825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58162024-02-15T02:59:47.615Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521456649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58172024-02-15T02:59:47.615Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521502643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58182024-02-15T02:59:47.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521545927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58192024-02-15T02:59:47.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521583683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58202024-02-15T02:59:47.615Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521621478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58212024-02-15T02:59:47.615Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521657104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58222024-02-15T02:59:47.615Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58232024-02-15T02:59:47.616ZSet Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521697548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58242024-02-15T02:59:47.616Z{"msg":"25aefeed-d4ac-4c0e-a840-3dbc7e5249df is now active with session: 6f925db0-95cd-4580-b7d8-9e8b8115191d","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521733334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58252024-02-15T02:59:47.616Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.521755621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343,"session_id":"6f925db0-95cd-4580-b7d8-9e8b8115191d"}
58262024-02-15T02:59:47.616Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:46.52176864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343}
58272024-02-15T02:59:47.617ZLoading write count information from file "/tmp/test_repair_verify.data"
58282024-02-15T02:59:47.617ZRun Repair workload
58292024-02-15T02:59:47.617Z01/30 Write block 308 len 40960 data: 6 7 8 7 6 7 5 4 4 4
58302024-02-15T02:59:47.617Z02/30 Flush
58312024-02-15T02:59:47.617Z03/30 Write block 200 len 36864 data: 3 3 2 2 2 3 5 5 5
58322024-02-15T02:59:47.637Z04/30 Write block 47 len 12288 data: 7 6 5
58332024-02-15T02:59:47.637Z05/30 Read block 267 len 16384
58342024-02-15T02:59:47.637Z06/30 Write block 331 len 20480 data: 7 8 7 8 9
58352024-02-15T02:59:47.637Z07/30 Flush
58362024-02-15T02:59:47.637Z08/30 Write block 383 len 4096 data: 6
58372024-02-15T02:59:47.637Z09/30 Read block 238 len 12288
58382024-02-15T02:59:47.637Z10/30 Read block 345 len 28672
58392024-02-15T02:59:47.637Z11/30 Write block 90 len 8192 data: 3 3
58402024-02-15T02:59:47.637Z12/30 Write block 140 len 28672 data: 7 7 7 7 6 6 5
58412024-02-15T02:59:47.637Z13/30 Write block 420 len 8192 data: 2 3
58422024-02-15T02:59:47.638Z14/30 Read block 545 len 40960
58432024-02-15T02:59:47.638Z15/30 Write block 330 len 36864 data: 6 8 9 8 9 10 8 7 7
58442024-02-15T02:59:47.638Z16/30 Read block 369 len 32768
58452024-02-15T02:59:47.638Z17/30 Write block 333 len 16384 data: 9 10 11 9
58462024-02-15T02:59:47.638Z18/30 Read block 165 len 12288
58472024-02-15T02:59:47.639Z19/30 Write block 67 len 40960 data: 3 4 4 3 4 3 2 3 5 5
58482024-02-15T02:59:47.639Z20/30 Read block 180 len 24576
58492024-02-15T02:59:47.639Z21/30 Write block 577 len 20480 data: 7 6 4 4 3
58502024-02-15T02:59:47.639Z22/30 Read block 351 len 36864
58512024-02-15T02:59:47.639Z23/30 Read block 168 len 20480
58522024-02-15T02:59:47.639Z24/30 Flush
58532024-02-15T02:59:47.640Z25/30 Write block 212 len 28672 data: 5 5 6 5 4 5 5
58542024-02-15T02:59:47.640Z26/30 Write block 52 len 28672 data: 3 5 5 5 5 5 4
58552024-02-15T02:59:47.640Z27/30 Write block 351 len 36864 data: 6 6 6 5 4 5 5 5 5
58562024-02-15T02:59:47.640Z28/30 Write block 526 len 32768 data: 3 6 6 6 5 3 5 3
58572024-02-15T02:59:47.640Z29/30 Write block 380 len 20480 data: 5 6 6 7 7
58582024-02-15T02:59:47.641Z30/30 Read block 543 len 4096
58592024-02-15T02:59:47.641Z----------------------------------------------------------------
58602024-02-15T02:59:47.641Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
58612024-02-15T02:59:47.641ZStates: Active Active Active
58622024-02-15T02:59:47.641ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
58632024-02-15T02:59:47.643Z 1 Acked 1000 Write 10 Done Done Sent false
58642024-02-15T02:59:47.643Z 2 Acked 1001 Flush 0 Done Done Sent false
58652024-02-15T02:59:47.643Z 3 Acked 1002 Write 9 Done Done Sent false
58662024-02-15T02:59:47.643Z 4 Acked 1003 Write 3 Done Done Sent false
58672024-02-15T02:59:47.644Z 5 Acked 1004 Read 4 Done Done Sent false
58682024-02-15T02:59:47.644Z 6 Acked 1005 Write 5 Done Done Sent false
58692024-02-15T02:59:47.644Z 7 Acked 1006 Flush 0 Done Done Sent false
58702024-02-15T02:59:47.644Z 8 Acked 1007 Write 1 Done Done Sent false
58712024-02-15T02:59:47.644Z 9 Acked 1008 Read 3 Done Done Sent false
58722024-02-15T02:59:47.645Z 10 Acked 1009 Read 7 Done Done Sent false
58732024-02-15T02:59:47.645Z 11 Acked 1010 Write 2 Done Done Sent false
58742024-02-15T02:59:47.645Z 12 Acked 1011 Write 7 Done Done Sent false
58752024-02-15T02:59:47.645Z 13 Acked 1012 Write 2 Done Done Sent false
58762024-02-15T02:59:47.645Z 14 Acked 1013 Read 10 Done Done Sent false
58772024-02-15T02:59:47.646Z 15 Acked 1014 Write 9 Done Done Sent false
58782024-02-15T02:59:47.646Z 16 Acked 1015 Read 8 Done Done Sent false
58792024-02-15T02:59:47.646Z 17 Acked 1016 Write 4 Done Done Sent false
58802024-02-15T02:59:47.646Z 18 Acked 1017 Read 3 Done Done Sent false
58812024-02-15T02:59:47.646Z 19 Acked 1018 Write 10 Done Done Sent false
58822024-02-15T02:59:47.646Z 20 Acked 1019 Read 6 Done Done Sent false
58832024-02-15T02:59:47.646Z 21 Acked 1020 Write 5 Done Done Sent false
58842024-02-15T02:59:47.646Z 22 Acked 1021 Read 9 Done Done Sent false
58852024-02-15T02:59:47.647Z 23 Acked 1022 Read 5 Done Done Sent false
58862024-02-15T02:59:47.647Z 24 Acked 1023 Flush 0 Done Done Sent false
58872024-02-15T02:59:47.647Z 25 Acked 1024 Write 7 Done Done Sent false
58882024-02-15T02:59:47.647Z 26 Acked 1025 Write 7 Done Done Sent false
58892024-02-15T02:59:47.647Z 27 Acked 1026 Write 9 Done Done Sent false
58902024-02-15T02:59:47.647Z 28 Acked 1027 Write 8 Done Done Sent false
58912024-02-15T02:59:47.648Z 29 Acked 1028 Write 5 Done Done Sent false
58922024-02-15T02:59:47.648Z 30 Acked 1029 Read 1 Done Done Sent false
58932024-02-15T02:59:47.648Z STATES DS:0 DS:1 DS:2 TOTAL
58942024-02-15T02:59:47.648Z New 0 0 0 0
58952024-02-15T02:59:47.648Z Sent 0 0 30 30
58962024-02-15T02:59:47.648Z Done 30 30 0 60
58972024-02-15T02:59:47.649Z Skipped 0 0 0 0
58982024-02-15T02:59:47.649Z Error 0 0 0 0
58992024-02-15T02:59:47.649ZLast Flush: 1023 1023 0
59002024-02-15T02:59:47.649ZDownstairs last five completed:
59012024-02-15T02:59:47.649ZUpstairs last five completed: 30 29 28 27 26
59022024-02-15T02:59:47.649Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:46.555992251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1343}
59032024-02-15T02:59:47.650ZWrote out file "/tmp/test_repair_verify.data"
59042024-02-15T02:59:48.095Z
59052024-02-15T02:59:48.095ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59062024-02-15T02:59:48.095ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59072024-02-15T02:59:48.096Z 0 000-019 49 49 49 90 90 90 F F F
59082024-02-15T02:59:48.096Z 1 020-039 49 49 49 90 90 90 F F F
59092024-02-15T02:59:48.096Z 2 040-059 52 52 48 96 96 89 T T F <---
59102024-02-15T02:59:48.096Z 3 060-079 52 52 44 97 97 83 F F F <---
59112024-02-15T02:59:48.096Z 4 080-099 52 52 45 97 97 84 F F F <---
59122024-02-15T02:59:48.096Z 5 100-119 51 51 51 94 94 94 F F F
59132024-02-15T02:59:48.096Z 6 120-139 51 51 51 94 94 94 F F F
59142024-02-15T02:59:48.097Z 7 140-159 52 52 50 97 97 93 F F F <---
59152024-02-15T02:59:48.097Z 8 160-179 50 50 50 91 91 91 F F F
59162024-02-15T02:59:48.097Z 9 180-199 51 51 51 94 94 94 F F F
59172024-02-15T02:59:48.097Z 10 200-219 52 52 44 96 96 83 T T F <---
59182024-02-15T02:59:48.098Z 11 220-239 50 50 50 93 93 93 F F F
59192024-02-15T02:59:48.098Z 12 240-259 46 46 46 87 87 87 F F F
59202024-02-15T02:59:48.098Z 13 260-279 44 44 44 83 83 83 F F F
59212024-02-15T02:59:48.098Z 14 280-299 49 49 49 90 90 90 F F F
59222024-02-15T02:59:48.098Z 15 300-319 52 52 49 95 95 90 F F F <---
59232024-02-15T02:59:48.099Z 16 320-339 52 52 48 97 97 89 F F F <---
59242024-02-15T02:59:48.099Z 17 340-359 50 50 50 92 92 92 T T F <---
59252024-02-15T02:59:48.099Z 18 360-379 51 51 51 94 94 94 F F F
59262024-02-15T02:59:48.099Z 19 380-399 52 52 48 97 97 89 T T F <---
59272024-02-15T02:59:48.099Z 20 400-419 41 41 41 78 78 78 F F F
59282024-02-15T02:59:48.100Z 21 420-439 52 52 47 97 97 88 F F F <---
59292024-02-15T02:59:48.100Z 22 440-459 50 50 50 92 92 92 F F F
59302024-02-15T02:59:48.100Z 23 460-479 48 48 48 89 89 89 F F F
59312024-02-15T02:59:48.100Z 24 480-499 50 50 50 92 92 92 F F F
59322024-02-15T02:59:48.101Z 25 500-519 50 50 50 91 91 91 F F F
59332024-02-15T02:59:48.102Z 26 520-539 44 44 44 83 83 83 T T F <---
59342024-02-15T02:59:48.102Z 27 540-559 50 50 50 93 93 93 F F F
59352024-02-15T02:59:48.102Z 28 560-579 52 52 50 97 97 93 F F F <---
59362024-02-15T02:59:48.103Z 29 580-599 52 52 49 97 97 90 F F F <---
59372024-02-15T02:59:48.103ZMax gen: 52, Max flush: 97
59382024-02-15T02:59:48.104ZError: Difference in extent metadata found!
59392024-02-15T02:59:48.105ZA Difference in extent metadata is expected here
59402024-02-15T02:59:48.105ZOn loop 26
59412024-02-15T02:59:48.105Z
59422024-02-15T02:59:48.106ZVerifying data now
59432024-02-15T02:59:52.307ZNo change after verify
59442024-02-15T02:59:52.307ZLoop: 26 Downstairs dump after verify (and repair):
59452024-02-15T02:59:52.332ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59462024-02-15T02:59:52.333Z 0 000-019 49 49 49 90 90 90 F F F
59472024-02-15T02:59:52.333Z 1 020-039 49 49 49 90 90 90 F F F
59482024-02-15T02:59:52.333Z 2 040-059 53 53 53 98 98 98 F F F
59492024-02-15T02:59:52.333Z 3 060-079 52 52 52 97 97 97 F F F
59502024-02-15T02:59:52.333Z 4 080-099 52 52 52 97 97 97 F F F
59512024-02-15T02:59:52.333Z 5 100-119 51 51 51 94 94 94 F F F
59522024-02-15T02:59:52.333Z 6 120-139 51 51 51 94 94 94 F F F
59532024-02-15T02:59:52.333Z 7 140-159 52 52 52 97 97 97 F F F
59542024-02-15T02:59:52.333Z 8 160-179 50 50 50 91 91 91 F F F
59552024-02-15T02:59:52.334Z 9 180-199 51 51 51 94 94 94 F F F
59562024-02-15T02:59:52.334Z 10 200-219 53 53 53 98 98 98 F F F
59572024-02-15T02:59:52.334Z 11 220-239 50 50 50 93 93 93 F F F
59582024-02-15T02:59:52.334Z 12 240-259 46 46 46 87 87 87 F F F
59592024-02-15T02:59:52.334Z 13 260-279 44 44 44 83 83 83 F F F
59602024-02-15T02:59:52.334Z 14 280-299 49 49 49 90 90 90 F F F
59612024-02-15T02:59:52.334Z 15 300-319 52 52 52 95 95 95 F F F
59622024-02-15T02:59:52.335Z 16 320-339 52 52 52 97 97 97 F F F
59632024-02-15T02:59:52.335Z 17 340-359 53 53 53 98 98 98 F F F
59642024-02-15T02:59:52.335Z 18 360-379 51 51 51 94 94 94 F F F
59652024-02-15T02:59:52.335Z 19 380-399 53 53 53 98 98 98 F F F
59662024-02-15T02:59:52.335Z 20 400-419 41 41 41 78 78 78 F F F
59672024-02-15T02:59:52.335Z 21 420-439 52 52 52 97 97 97 F F F
59682024-02-15T02:59:52.335Z 22 440-459 50 50 50 92 92 92 F F F
59692024-02-15T02:59:52.335Z 23 460-479 48 48 48 89 89 89 F F F
59702024-02-15T02:59:52.336Z 24 480-499 50 50 50 92 92 92 F F F
59712024-02-15T02:59:52.336Z 25 500-519 50 50 50 91 91 91 F F F
59722024-02-15T02:59:52.336Z 26 520-539 53 53 53 98 98 98 F F F
59732024-02-15T02:59:52.336Z 27 540-559 50 50 50 93 93 93 F F F
59742024-02-15T02:59:52.336Z 28 560-579 52 52 52 97 97 97 F F F
59752024-02-15T02:59:52.336Z 29 580-599 52 52 52 97 97 97 F F F
59762024-02-15T02:59:52.336ZMax gen: 53, Max flush: 98
59772024-02-15T02:59:52.337Z
59782024-02-15T02:59:52.337ZBegin loop 27
59792024-02-15T02:59:52.337ZDownstairs to restart: 1
59802024-02-15T02:59:52.395Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.261894376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351}
59812024-02-15T02:59:52.395Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262196668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351}
59822024-02-15T02:59:52.395Z{"msg":"Crucible runtime is spawned
59832024-02-15T02:59:52.396ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262235493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351}
59842024-02-15T02:59:52.396Z{"msg":"Crucible 932b52a4-9d81-4b90-85a6-43636691b519 has session id: 7189a1e2-3b4a-4fd9-bbc8-74e2594136cb","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262267489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59852024-02-15T02:59:52.396Z{"msg":"Upstairs opts: Upstairs UUID: 932b52a4-9d81-4b90-85a6-43636691b519, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262320243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59862024-02-15T02:59:52.396Z{"msg":"Crucible stats registered with UUID: 932b52a4-9d81-4b90-85a6-43636691b519","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.26234519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59872024-02-15T02:59:52.396Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262379765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351}
59882024-02-15T02:59:52.396Z{"msg":"932b52a4-9d81-4b90-85a6-43636691b519 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262396543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59892024-02-15T02:59:52.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.26242381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59902024-02-15T02:59:52.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262472754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59912024-02-15T02:59:52.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262509559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59922024-02-15T02:59:52.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262537006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59932024-02-15T02:59:52.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262595798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59942024-02-15T02:59:52.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262728651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59952024-02-15T02:59:52.397Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262816221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59962024-02-15T02:59:52.397Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262890341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59972024-02-15T02:59:52.397Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262950944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59982024-02-15T02:59:52.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262974711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
59992024-02-15T02:59:52.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.262999348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60002024-02-15T02:59:52.397Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:51.263020825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60012024-02-15T02:59:52.397Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:51.263048482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60022024-02-15T02:59:52.398Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:51.263079747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60032024-02-15T02:59:52.398Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.263117573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60042024-02-15T02:59:52.398Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.263149719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60052024-02-15T02:59:52.398Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.263190144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60062024-02-15T02:59:52.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.263533511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60072024-02-15T02:59:52.398Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.263581725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60082024-02-15T02:59:52.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.263635138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60092024-02-15T02:59:52.398Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.263680762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60102024-02-15T02:59:52.398Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264200947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60112024-02-15T02:59:52.398Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264281017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60122024-02-15T02:59:52.399Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264807691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60132024-02-15T02:59:52.399Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264850336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60142024-02-15T02:59:52.399Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264874173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60152024-02-15T02:59:52.399Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264934785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60162024-02-15T02:59:52.399Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264965281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60172024-02-15T02:59:52.399Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:51.264985279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60182024-02-15T02:59:53.331Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.264728914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60192024-02-15T02:59:53.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.264824752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60202024-02-15T02:59:53.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.26571999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60212024-02-15T02:59:53.339Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.265765114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60222024-02-15T02:59:53.339Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.266261332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60232024-02-15T02:59:53.339Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.266881314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60242024-02-15T02:59:53.339Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.266930978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60252024-02-15T02:59:53.339Z{"msg":"[0]R flush_numbers[0..12]: [90, 90, 98, 97, 97, 94, 94, 97, 91, 94, 98, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.266952165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60262024-02-15T02:59:53.340Z{"msg":"[0]R generation[0..12]: [49, 49, 53, 52, 52, 51, 51, 52, 50, 51, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.266971513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60272024-02-15T02:59:53.340Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.266991411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60282024-02-15T02:59:53.340Z{"msg":"[1]R flush_numbers[0..12]: [90, 90, 98, 97, 97, 94, 94, 97, 91, 94, 98, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267019087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60292024-02-15T02:59:53.340Z{"msg":"[1]R generation[0..12]: [49, 49, 53, 52, 52, 51, 51, 52, 50, 51, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267037365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60302024-02-15T02:59:53.340Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267063882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60312024-02-15T02:59:53.340Z{"msg":"[2]R flush_numbers[0..12]: [90, 90, 98, 97, 97, 94, 94, 97, 91, 94, 98, 93]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267081889Z","hostname":"Wait for a query_work_queue command to finish before sending IO
60322024-02-15T02:59:53.340Zip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60332024-02-15T02:59:53.340Z{"msg":"[2]R generation[0..12]: [49, 49, 53, 52, 52, 51, 51, 52, 50, 51, 53, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267110776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60342024-02-15T02:59:53.340Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267128583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60352024-02-15T02:59:53.340Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.26715323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60362024-02-15T02:59:53.340Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267171598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60372024-02-15T02:59:53.340Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267187956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60382024-02-15T02:59:53.341Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267212283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60392024-02-15T02:59:53.341Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267229271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60402024-02-15T02:59:53.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267252638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60412024-02-15T02:59:53.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267269546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60422024-02-15T02:59:53.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267300682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60432024-02-15T02:59:53.341Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.26731781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60442024-02-15T02:59:53.341Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267341577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60452024-02-15T02:59:53.341Z{"msg":"932b52a4-9d81-4b90-85a6-43636691b519 is now active with session: 7189a1e2-3b4a-4fd9-bbc8-74e2594136cb","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267358015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60462024-02-15T02:59:53.341Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.267380782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351,"session_id":"7189a1e2-3b4a-4fd9-bbc8-74e2594136cb"}
60472024-02-15T02:59:53.341Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:52.26739642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351}
60482024-02-15T02:59:53.341ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60492024-02-15T02:59:53.341ZLoading write count information from file "/tmp/test_repair_verify.data"
60502024-02-15T02:59:53.341ZRun Repair workload
60512024-02-15T02:59:53.342Z01/30 Write block 135 len 20480 data: 6 6 5 5 5
60522024-02-15T02:59:53.342Z02/30 Read block 130 len 4096
60532024-02-15T02:59:53.342Z03/30 Read block 403 len 8192
60542024-02-15T02:59:53.342Z04/30 Read block 184 len 20480
60552024-02-15T02:59:53.342Z05/30 Write block 174 len 20480 data: 4 4 3 4 4
60562024-02-15T02:59:53.358Z06/30 Read block 562 len 36864
60572024-02-15T02:59:53.358Z07/30 Read block 486 len 32768
60582024-02-15T02:59:53.358Z08/30 Read block 2 len 24576
60592024-02-15T02:59:53.358Z09/30 Write block 94 len 8192 data: 6 6
60602024-02-15T02:59:53.358Z10/30 Write block 221 len 16384 data: 8 8 6 6
60612024-02-15T02:59:53.358Z11/30 Read block 557 len 20480
60622024-02-15T02:59:53.358Z12/30 Read block 342 len 24576
60632024-02-15T02:59:53.358Z13/30 Read block 463 len 24576
60642024-02-15T02:59:53.358Z14/30 Write block 546 len 40960 data: 4 3 4 3 3 3 3 3 3 4
60652024-02-15T02:59:53.359Z15/30 Read block 11 len 24576
60662024-02-15T02:59:53.359Z16/30 Read block 510 len 36864
60672024-02-15T02:59:53.359Z17/30 Flush
60682024-02-15T02:59:53.359Z18/30 Write block 7 len 16384 data: 6 7 7 8
60692024-02-15T02:59:53.359Z19/30 Read block 174 len 8192
60702024-02-15T02:59:53.359Z20/30 Write block 436 len 8192 data: 6 7
60712024-02-15T02:59:53.359Z21/30 Read block 368 len 8192
60722024-02-15T02:59:53.359Z22/30 Write block 405 len 24576 data: 5 6 6 6 7 7
60732024-02-15T02:59:53.359Z23/30 Flush
60742024-02-15T02:59:53.359Z24/30 Write block 426 len 8192 data: 5 5
60752024-02-15T02:59:53.359Z25/30 Read block 79 len 40960
60762024-02-15T02:59:53.364Z26/30 Read block 543 len 4096
60772024-02-15T02:59:53.364Z27/30 Write block 166 len 8192 data: 5 5
60782024-02-15T02:59:53.364Z28/30 Read block 7 len 40960
60792024-02-15T02:59:53.364Z29/30 Write block 538 len 24576 data: 5 5 4 5 7 7
60802024-02-15T02:59:53.364Z30/30 Read block 86 len 16384
60812024-02-15T02:59:53.364Z----------------------------------------------------------------
60822024-02-15T02:59:53.364Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
60832024-02-15T02:59:53.364ZStates: Active Active Active
60842024-02-15T02:59:53.365ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
60852024-02-15T02:59:53.365Z 1 Acked 1000 Write 5 Done Sent Done false
60862024-02-15T02:59:53.365Z 2 Acked 1001 Read 1 Done Sent Done false
60872024-02-15T02:59:53.365Z 3 Acked 1002 Read 2 Done Sent Done false
60882024-02-15T02:59:53.365Z 4 Acked 1003 Read 5 Done Sent Done false
60892024-02-15T02:59:53.365Z 5 Acked 1004 Write 5 Done Sent Done false
60902024-02-15T02:59:53.365Z 6 Acked 1005 Read 9 Done Sent Done false
60912024-02-15T02:59:53.365Z 7 Acked 1006 Read 8 Done Sent Done false
60922024-02-15T02:59:53.365Z 8 Acked 1007 Read 6 Done Sent Done false
60932024-02-15T02:59:53.365Z 9 Acked 1008 Write 2 Done Sent Done false
60942024-02-15T02:59:53.365Z 10 Acked 1009 Write 4 Done Sent Done false
60952024-02-15T02:59:53.365Z 11 Acked 1010 Read 5 Done Sent Done false
60962024-02-15T02:59:53.366Z 12 Acked 1011 Read 6 Done Sent Done false
60972024-02-15T02:59:53.366Z 13 Acked 1012 Read 6 Done Sent Done false
60982024-02-15T02:59:53.366Z 14 Acked 1013 Write 10 Done Sent Done false
60992024-02-15T02:59:53.366Z 15 Acked 1014 Read 6 Done Sent Done false
61002024-02-15T02:59:53.366Z 16 Acked 1015 Read 9 Done Sent Done false
61012024-02-15T02:59:53.366Z 17 Acked 1016 Flush 0 Done Sent Done false
61022024-02-15T02:59:53.366Z 18 Acked 1017 Write 4 Done Sent Done false
61032024-02-15T02:59:53.366Z 19 Acked 1018 Read 2 Done Sent Done false
61042024-02-15T02:59:53.366Z 20 Acked 1019 Write 2 Done Sent Done false
61052024-02-15T02:59:53.366Z 21 Acked 1020 Read 2 Done Sent Done false
61062024-02-15T02:59:53.366Z 22 Acked 1021 Write 6 Done Sent Done false
61072024-02-15T02:59:53.366Z 23 Acked 1022 Flush 0 Done Sent Done false
61082024-02-15T02:59:53.366Z 24 Acked 1023 Write 2 Done Sent Done false
61092024-02-15T02:59:53.367Z 25 Acked 1024 Read 10 Done Sent Done false
61102024-02-15T02:59:53.367Z 26 Acked 1025 Read 1 Done Sent Done false
61112024-02-15T02:59:53.367Z 27 Acked 1026 Write 2 Done Sent Done false
61122024-02-15T02:59:53.367Z 28 Acked 1027 Read 10 Done Sent Done false
61132024-02-15T02:59:53.367Z 29 Acked 1028 Write 6 Done Sent Done false
61142024-02-15T02:59:53.367Z 30 Acked 1029 Read 4 Done Sent Sent false
61152024-02-15T02:59:53.367Z STATES DS:0 DS:1 DS:2 TOTAL
61162024-02-15T02:59:53.367Z New 0 0 0 0
61172024-02-15T02:59:53.367Z Sent 0 30 1 31
61182024-02-15T02:59:53.367Z Done 30 0 29 59
61192024-02-15T02:59:53.367Z Skipped 0 0 0 0
61202024-02-15T02:59:53.368Z Error 0 0 0 0
61212024-02-15T02:59:53.368ZLast Flush: 1022 0 1022
61222024-02-15T02:59:53.368ZDownstairs last five completed:
61232024-02-15T02:59:53.368ZUpstairs last five completed: 30 29 28 27 26
61242024-02-15T02:59:53.368Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:52.298611789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1351}
61252024-02-15T02:59:53.368ZWrote out file "/tmp/test_repair_verify.data"
61262024-02-15T02:59:53.373Z
61272024-02-15T02:59:53.373ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61282024-02-15T02:59:53.391ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61292024-02-15T02:59:53.391Z 0 000-019 54 49 54 100 90 100 F F F <---
61302024-02-15T02:59:53.391Z 1 020-039 49 49 49 90 90 90 F F F
61312024-02-15T02:59:53.391Z 2 040-059 53 53 53 98 98 98 F F F
61322024-02-15T02:59:53.391Z 3 060-079 52 52 52 97 97 97 F F F
61332024-02-15T02:59:53.391Z 4 080-099 54 52 54 99 97 99 F F F <---
61342024-02-15T02:59:53.391Z 5 100-119 51 51 51 94 94 94 F F F
61352024-02-15T02:59:53.391Z 6 120-139 54 51 54 99 94 99 F F F <---
61362024-02-15T02:59:53.391Z 7 140-159 52 52 52 97 97 97 F F F
61372024-02-15T02:59:53.391Z 8 160-179 54 50 54 99 91 99 T F T <---
61382024-02-15T02:59:53.391Z 9 180-199 51 51 51 94 94 94 F F F
61392024-02-15T02:59:53.392Z 10 200-219 53 53 53 98 98 98 F F F
61402024-02-15T02:59:53.395Z 11 220-239 54 50 54 99 93 99 F F F <---
61412024-02-15T02:59:53.396Z 12 240-259 46 46 46 87 87 87 F F F
61422024-02-15T02:59:53.396Z 13 260-279 44 44 44 83 83 83 F F F
61432024-02-15T02:59:53.396Z 14 280-299 49 49 49 90 90 90 F F F
61442024-02-15T02:59:53.396Z 15 300-319 52 52 52 95 95 95 F F F
61452024-02-15T02:59:53.396Z 16 320-339 52 52 52 97 97 97 F F F
61462024-02-15T02:59:53.396Z 17 340-359 53 53 53 98 98 98 F F F
61472024-02-15T02:59:53.396Z 18 360-379 51 51 51 94 94 94 F F F
61482024-02-15T02:59:53.396Z 19 380-399 53 53 53 98 98 98 F F F
61492024-02-15T02:59:53.396Z 20 400-419 54 41 54 100 78 100 F F F <---
61502024-02-15T02:59:53.396Z 21 420-439 54 52 54 100 97 100 T F T <---
61512024-02-15T02:59:53.396Z 22 440-459 50 50 50 92 92 92 F F F
61522024-02-15T02:59:53.396Z 23 460-479 48 48 48 89 89 89 F F F
61532024-02-15T02:59:53.396Z 24 480-499 50 50 50 92 92 92 F F F
61542024-02-15T02:59:53.396Z 25 500-519 50 50 50 91 91 91 F F F
61552024-02-15T02:59:53.396Z 26 520-539 53 53 53 98 98 98 T F T <---
61562024-02-15T02:59:53.396Z 27 540-559 54 50 54 99 93 99 T F T <---
61572024-02-15T02:59:53.396Z 28 560-579 52 52 52 97 97 97 F F F
61582024-02-15T02:59:53.396Z 29 580-599 52 52 52 97 97 97 F F F
61592024-02-15T02:59:53.396ZMax gen: 54, Max flush: 100
61602024-02-15T02:59:53.396ZError: Difference in extent metadata found!
61612024-02-15T02:59:53.396ZA Difference in extent metadata is expected here
61622024-02-15T02:59:53.396ZOn loop 27
61632024-02-15T02:59:53.396Z
61642024-02-15T02:59:53.459ZVerifying data now
61652024-02-15T02:59:57.881ZNo change after verify
61662024-02-15T02:59:57.882ZLoop: 27 Downstairs dump after verify (and repair):
61672024-02-15T02:59:57.926ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61682024-02-15T02:59:57.926Z 0 000-019 54 54 54 100 100 100 F F F
61692024-02-15T02:59:57.926Z 1 020-039 49 49 49 90 90 90 F F F
61702024-02-15T02:59:57.926Z 2 040-059 53 53 53 98 98 98 F F F
61712024-02-15T02:59:57.926Z 3 060-079 52 52 52 97 97 97 F F F
61722024-02-15T02:59:57.926Z 4 080-099 54 54 54 99 99 99 F F F
61732024-02-15T02:59:57.927Z 5 100-119 51 51 51 94 94 94 F F F
61742024-02-15T02:59:57.927Z 6 120-139 54 54 54 99 99 99 F F F
61752024-02-15T02:59:57.927Z 7 140-159 52 52 52 97 97 97 F F F
61762024-02-15T02:59:57.927Z 8 160-179 55 55 55 101 101 101 F F F
61772024-02-15T02:59:57.927Z 9 180-199 51 51 51 94 94 94 F F F
61782024-02-15T02:59:57.927Z 10 200-219 53 53 53 98 98 98 F F F
61792024-02-15T02:59:57.927Z 11 220-239 54 54 54 99 99 99 F F F
61802024-02-15T02:59:57.927Z 12 240-259 46 46 46 87 87 87 F F F
61812024-02-15T02:59:57.928Z 13 260-279 44 44 44 83 83 83 F F F
61822024-02-15T02:59:57.928Z 14 280-299 49 49 49 90 90 90 F F F
61832024-02-15T02:59:57.928Z 15 300-319 52 52 52 95 95 95 F F F
61842024-02-15T02:59:57.928Z 16 320-339 52 52 52 97 97 97 F F F
61852024-02-15T02:59:57.928Z 17 340-359 53 53 53 98 98 98 F F F
61862024-02-15T02:59:57.928Z 18 360-379 51 51 51 94 94 94 F F F
61872024-02-15T02:59:57.928Z 19 380-399 53 53 53 98 98 98 F F F
61882024-02-15T02:59:57.928Z 20 400-419 54 54 54 100 100 100 F F F
61892024-02-15T02:59:57.928Z 21 420-439 55 55 55 101 101 101 F F F
61902024-02-15T02:59:57.928Z 22 440-459 50 50 50 92 92 92 F F F
61912024-02-15T02:59:57.928Z 23 460-479 48 48 48 89 89 89 F F F
61922024-02-15T02:59:57.928Z 24 480-499 50 50 50 92 92 92 F F F
61932024-02-15T02:59:57.928Z 25 500-519 50 50 50 91 91 91 F F F
61942024-02-15T02:59:57.929Z 26 520-539 55 55 55 101 101 101 F F F
61952024-02-15T02:59:57.929Z 27 540-559 55 55 55 101 101 101 F F F
61962024-02-15T02:59:57.929Z 28 560-579 52 52 52 97 97 97 F F F
61972024-02-15T02:59:57.929Z 29 580-599 52 52 52 97 97 97 F F F
61982024-02-15T02:59:57.929ZMax gen: 55, Max flush: 101
61992024-02-15T02:59:57.929Z
62002024-02-15T02:59:57.929ZBegin loop 28
62012024-02-15T02:59:57.929ZDownstairs to restart: 2
62022024-02-15T02:59:57.939Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.853990244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359}
62032024-02-15T02:59:57.939Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854290756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359}
62042024-02-15T02:59:57.939Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854308064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359}
62052024-02-15T02:59:57.939Z{"msg":"Crucible runtime is spawned
62062024-02-15T02:59:57.939ZCrucible ebaf0dc4-0fc1-4242-96cd-02bb7bb64d2a has session id: 83782e10-e72c-4913-a13d-181e0f2f6df0","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.85434398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62072024-02-15T02:59:57.939Z{"msg":"Upstairs opts: Upstairs UUID: ebaf0dc4-0fc1-4242-96cd-02bb7bb64d2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854386784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62082024-02-15T02:59:57.939Z{"msg":"Crucible stats registered with UUID: ebaf0dc4-0fc1-4242-96cd-02bb7bb64d2a","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854427709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62092024-02-15T02:59:57.940Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854448866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359}
62102024-02-15T02:59:57.940Z{"msg":"ebaf0dc4-0fc1-4242-96cd-02bb7bb64d2a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854506159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62112024-02-15T02:59:57.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854539785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62122024-02-15T02:59:57.940Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854556453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62132024-02-15T02:59:57.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854586909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62142024-02-15T02:59:57.940Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854608506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62152024-02-15T02:59:57.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854638953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62162024-02-15T02:59:57.940Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854655101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62172024-02-15T02:59:57.940Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854685437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62182024-02-15T02:59:57.940Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854710654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62192024-02-15T02:59:57.940Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854736121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62202024-02-15T02:59:57.941Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:56.854879023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62212024-02-15T02:59:57.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854916528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62222024-02-15T02:59:57.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.854941965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62232024-02-15T02:59:57.941Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:56.854969221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62242024-02-15T02:59:57.941Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:56.855034503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62252024-02-15T02:59:57.941Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.855080237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62262024-02-15T02:59:57.941Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.855116563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62272024-02-15T02:59:57.941Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.855143219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62282024-02-15T02:59:57.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.855611441Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62292024-02-15T02:59:57.941Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.855648386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62302024-02-15T02:59:57.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.855664144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62312024-02-15T02:59:57.941Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.85569777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62322024-02-15T02:59:57.942Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.85625806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62332024-02-15T02:59:57.942Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.856308564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62342024-02-15T02:59:57.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.856933945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62352024-02-15T02:59:57.942Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.856969791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62362024-02-15T02:59:57.942Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.856986668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62372024-02-15T02:59:57.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.857006576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62382024-02-15T02:59:57.942Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.857033183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62392024-02-15T02:59:57.942Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:56.857047741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62402024-02-15T02:59:58.940Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.856476455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62412024-02-15T02:59:58.940Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.856568634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62422024-02-15T02:59:58.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.857453473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62432024-02-15T02:59:58.969Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.857504377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62442024-02-15T02:59:58.969Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858005844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62452024-02-15T02:59:58.969Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.85859363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62462024-02-15T02:59:58.970Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858643634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62472024-02-15T02:59:58.970Z{"msg":"[0]R flush_numbers[0..12]: [100, 90, 98, 97, 99, 94, 99, 97, 101, 94, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858680189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62482024-02-15T02:59:58.970Z{"msg":"[0]R generation[0..12]: [54, 49, 53, 52, 54, 51, 54, 52, 55, 51, 53, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858714055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62492024-02-15T02:59:58.970Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858733373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62502024-02-15T02:59:58.970Z{"msg":"[1]R flush_numbers[0..12]: [100, 90, 98, 97, 99, 94, 99, 97, 101, 94, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858759789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62512024-02-15T02:59:58.970Z{"msg":"[1]R generation[0..12]: [54, 49, 53, 52, 54, 51, 54, 52, 55, 51, 53, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858777667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62522024-02-15T02:59:58.970ZWait for a query_work_queue command to finish before sending IO
62532024-02-15T02:59:58.970Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858812033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62542024-02-15T02:59:58.971Z{"msg":"[2]R flush_numbers[0..12]: [100, 90, 98, 97, 99, 94, 99, 97, 101, 94, 98, 99]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.85883798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62552024-02-15T02:59:58.971Z{"msg":"[2]R generation[0..12]: [54, 49, 53, 52, 54, 51, 54, 52, 55, 51, 53, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858855657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62562024-02-15T02:59:58.971Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858892033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62572024-02-15T02:59:58.971Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858933028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62582024-02-15T02:59:58.971Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.858966823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62592024-02-15T02:59:58.971Z{"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859012948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62602024-02-15T02:59:58.971Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859060012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62612024-02-15T02:59:58.972Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
62622024-02-15T02:59:58.972Z,"time":"2024-02-15T02:59:57.859106726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62632024-02-15T02:59:58.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859137542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62642024-02-15T02:59:58.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859167678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62652024-02-15T02:59:58.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859185036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62662024-02-15T02:59:58.972Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859206253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62672024-02-15T02:59:58.973Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859219242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62682024-02-15T02:59:58.973Z{"msg":"ebaf0dc4-0fc1-4242-96cd-02bb7bb64d2a is now active with session: 83782e10-e72c-4913-a13d-181e0f2f6df0","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859239219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62692024-02-15T02:59:58.973Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859251558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359,"session_id":"83782e10-e72c-4913-a13d-181e0f2f6df0"}
62702024-02-15T02:59:58.973Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-15T02:59:57.859272145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359}
62712024-02-15T02:59:58.973ZLoading write count information from file "/tmp/test_repair_verify.data"
62722024-02-15T02:59:58.974ZRun Repair workload
62732024-02-15T02:59:58.974Z01/30 Write block 32 len 28672 data: 5 5 5 6 5 5 6
62742024-02-15T02:59:58.974Z02/30 Read block 268 len 8192
62752024-02-15T02:59:58.974Z03/30 Flush
62762024-02-15T02:59:58.974Z04/30 Write block 60 len 16384 data: 3 3 4 4
62772024-02-15T02:59:58.974Z05/30 Write block 124 len 12288 data: 5 4 5
62782024-02-15T02:59:58.974Z06/30 Read block 344 len 36864
62792024-02-15T02:59:58.974Z07/30 Write block 55 len 40960 data: 6 6 6 5 3 4 4 5 5 5
62802024-02-15T02:59:58.974Z08/30 Read block 231 len 20480
62812024-02-15T02:59:58.974Z09/30 Write block 442 len 4096 data: 9
62822024-02-15T02:59:58.974Z10/30 Write block 238 len 36864 data: 4 4 5 5 6 6 6 7 6
62832024-02-15T02:59:58.974Z11/30 Read block 147 len 24576
62842024-02-15T02:59:58.975Z12/30 Write block 431 len 4096 data: 4
62852024-02-15T02:59:58.975Z13/30 Write block 591 len 20480 data: 6 4 4 6 4
62862024-02-15T02:59:58.975Z14/30 Read block 530 len 4096
62872024-02-15T02:59:58.975Z15/30 Read block 518 len 36864
62882024-02-15T02:59:58.975Z16/30 Read block 447 len 16384
62892024-02-15T02:59:58.975Z17/30 Write block 42 len 32768 data: 4 3 6 6 7 8 7 6
62902024-02-15T02:59:58.975Z18/30 Read block 302 len 12288
62912024-02-15T02:59:58.975Z19/30 Write block 26 len 24576 data: 2 3 4 4 6 6
62922024-02-15T02:59:58.975Z20/30 Write block 419 len 12288 data: 3 3 4
62932024-02-15T02:59:58.976Z21/30 Read block 14 len 4096
62942024-02-15T02:59:58.976Z22/30 Read block 468 len 32768
62952024-02-15T02:59:58.976Z23/30 Write block 144 len 16384 data: 7 7 6 7
62962024-02-15T02:59:58.976Z24/30 Write block 466 len 20480 data: 5 5 6 7 8
62972024-02-15T02:59:58.976Z25/30 Write block 593 len 24576 data: 5 7 5 3 3 3
62982024-02-15T02:59:58.976Z26/30 Write block 290 len 40960 data: 7 6 6 6 7 5 6 5 6 7
62992024-02-15T02:59:58.976Z27/30 Write block 575 len 40960 data: 7 8 8 7 5 5 4 5 4 4
63002024-02-15T02:59:58.976Z28/30 Write block 8 len 16384 data: 8 8 9 5
63012024-02-15T02:59:58.993Z29/30 Write block 492 len 28672 data: 6 6 7 7 7 7 7
63022024-02-15T02:59:58.993Z30/30 Read block 190 len 4096
63032024-02-15T02:59:58.993Z----------------------------------------------------------------
63042024-02-15T02:59:58.995Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:27
63052024-02-15T02:59:58.995ZStates: Active Active Active
63062024-02-15T02:59:58.995ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63072024-02-15T02:59:58.995Z 4 Acked 1003 Write 4 Done Done Done false
63082024-02-15T02:59:58.995Z 5 Acked 1004 Write 3 Done Done Sent false
63092024-02-15T02:59:58.996Z 6 Acked 1005 Read 9 Done Done Sent false
63102024-02-15T02:59:58.996Z 7 Acked 1006 Write 10 Done Done Sent false
63112024-02-15T02:59:58.996Z 8 Acked 1007 Read 5 Done Done Sent false
63122024-02-15T02:59:58.996Z 9 Acked 1008 Write 1 Done Done Sent false
63132024-02-15T02:59:58.996Z 10 Acked 1009 Write 9 Done Done Sent false
63142024-02-15T02:59:58.996Z 11 Acked 1010 Read 6 Done Done Sent false
63152024-02-15T02:59:58.996Z 12 Acked 1011 Write 1 Done Done Sent false
63162024-02-15T02:59:58.997Z 13 Acked 1012 Write 5 Done Done Sent false
63172024-02-15T02:59:58.997Z 14 Acked 1013 Read 1 Done Done Sent false
63182024-02-15T02:59:58.997Z 15 Acked 1014 Read 9 Done Done Sent false
63192024-02-15T02:59:58.997Z 16 Acked 1015 Read 4 Done Done Sent false
63202024-02-15T02:59:58.997Z 17 Acked 1016 Write 8 Done Done Sent false
63212024-02-15T02:59:58.997Z 18 Acked 1017 Read 3 Done Done Sent false
63222024-02-15T02:59:58.997Z 19 Acked 1018 Write 6 Done Done Sent false
63232024-02-15T02:59:58.998Z 20 Acked 1019 Write 3 Done Done Sent false
63242024-02-15T02:59:58.998Z 21 Acked 1020 Read 1 Done Done Sent false
63252024-02-15T02:59:58.998Z 22 Acked 1021 Read 8 Done Done Sent false
63262024-02-15T02:59:58.998Z 23 Acked 1022 Write 4 Done Done Sent false
63272024-02-15T02:59:58.998Z 24 Acked 1023 Write 5 Done Done Sent false
63282024-02-15T02:59:58.998Z 25 Acked 1024 Write 6 Done Done Sent false
63292024-02-15T02:59:58.998Z 26 Acked 1025 Write 10 Done Done Sent false
63302024-02-15T02:59:58.998Z 27 Acked 1026 Write 10 Done Done Sent false
63312024-02-15T02:59:58.998Z 28 Acked 1027 Write 4 Done Done Sent false
63322024-02-15T02:59:58.998Z 29 Acked 1028 Write 7 Done Done Sent false
63332024-02-15T02:59:58.998Z 30 Acked 1029 Read 1 Done Done Sent false
63342024-02-15T02:59:58.998Z STATES DS:0 DS:1 DS:2 TOTAL
63352024-02-15T02:59:58.998Z New 0 0 0 0
63362024-02-15T02:59:58.998Z Sent 0 0 26 26
63372024-02-15T02:59:58.998Z Done 27 27 1 55
63382024-02-15T02:59:58.998Z Skipped 0 0 0 0
63392024-02-15T02:59:58.998Z Error 0 0 0 0
63402024-02-15T02:59:58.998ZLast Flush: 1002 1002 1002
63412024-02-15T02:59:58.998ZDownstairs last five completed: 1002 1001 1000
63422024-02-15T02:59:58.998ZUpstairs last five completed: 30 29 28 27 26
63432024-02-15T02:59:58.998Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T02:59:57.887654459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1359}
63442024-02-15T02:59:58.998ZWrote out file "/tmp/test_repair_verify.data"
63452024-02-15T02:59:58.998Z
63462024-02-15T02:59:58.999ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63472024-02-15T02:59:58.999ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63482024-02-15T02:59:58.999Z 0 000-019 54 54 54 100 100 100 T T F <---
63492024-02-15T02:59:58.999Z 1 020-039 56 56 56 102 102 102 T T F <---
63502024-02-15T02:59:58.999Z 2 040-059 53 53 53 98 98 98 T T F <---
63512024-02-15T02:59:58.999Z 3 060-079 52 52 52 97 97 97 T T T
63522024-02-15T02:59:58.999Z 4 080-099 54 54 54 99 99 99 F F F
63532024-02-15T02:59:58.999Z 5 100-119 51 51 51 94 94 94 F F F
63542024-02-15T02:59:58.999Z 6 120-139 54 54 54 99 99 99 T T F <---
63552024-02-15T02:59:58.999Z 7 140-159 52 52 52 97 97 97 T T F <---
63562024-02-15T02:59:58.999Z 8 160-179 55 55 55 101 101 101 F F F
63572024-02-15T02:59:58.999Z 9 180-199 51 51 51 94 94 94 F F F
63582024-02-15T02:59:58.999Z 10 200-219 53 53 53 98 98 98 F F F
63592024-02-15T02:59:58.999Z 11 220-239 54 54 54 99 99 99 T T F <---
63602024-02-15T02:59:58.999Z 12 240-259 46 46 46 87 87 87 T T F <---
63612024-02-15T02:59:58.999Z 13 260-279 44 44 44 83 83 83 F F F
63622024-02-15T02:59:59.000Z 14 280-299 49 49 49 90 90 90 T T F <---
63632024-02-15T02:59:59.000Z 15 300-319 52 52 52 95 95 95 F F F
63642024-02-15T02:59:59.000Z 16 320-339 52 52 52 97 97 97 F F F
63652024-02-15T02:59:59.000Z 17 340-359 53 53 53 98 98 98 F F F
63662024-02-15T02:59:59.000Z 18 360-379 51 51 51 94 94 94 F F F
63672024-02-15T02:59:59.000Z 19 380-399 53 53 53 98 98 98 F F F
63682024-02-15T02:59:59.000Z 20 400-419 54 54 54 100 100 100 T T F <---
63692024-02-15T02:59:59.000Z 21 420-439 55 55 55 101 101 101 T T F <---
63702024-02-15T02:59:59.000Z 22 440-459 50 50 50 92 92 92 T T F <---
63712024-02-15T02:59:59.000Z 23 460-479 48 48 48 89 89 89 T T F <---
63722024-02-15T02:59:59.000Z 24 480-499 50 50 50 92 92 92 T T F <---
63732024-02-15T02:59:59.000Z 25 500-519 50 50 50 91 91 91 F F F
63742024-02-15T02:59:59.000Z 26 520-539 55 55 55 101 101 101 F F F
63752024-02-15T02:59:59.000Z 27 540-559 55 55 55 101 101 101 F F F
63762024-02-15T02:59:59.000Z 28 560-579 52 52 52 97 97 97 T T F <---
63772024-02-15T02:59:59.000Z 29 580-599 52 52 52 97 97 97 T T F <---
63782024-02-15T02:59:59.000ZMax gen: 56, Max flush: 102
63792024-02-15T02:59:59.000ZError: Difference in extent metadata found!
63802024-02-15T02:59:59.018ZA Difference in extent metadata is expected here
63812024-02-15T02:59:59.018ZOn loop 28
63822024-02-15T02:59:59.018Z
63832024-02-15T02:59:59.018ZVerifying data now
63842024-02-15T03:00:04.803ZNo change after verify
63852024-02-15T03:00:04.803ZLoop: 28 Downstairs dump after verify (and repair):
63862024-02-15T03:00:04.848ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63872024-02-15T03:00:04.849Z 0 000-019 57 57 57 103 103 103 F F F
63882024-02-15T03:00:04.849Z 1 020-039 57 57 57 103 103 103 F F F
63892024-02-15T03:00:04.849Z 2 040-059 57 57 57 103 103 103 F F F
63902024-02-15T03:00:04.849Z 3 060-079 57 57 57 103 103 103 F F F
63912024-02-15T03:00:04.849Z 4 080-099 54 54 54 99 99 99 F F F
63922024-02-15T03:00:04.849Z 5 100-119 51 51 51 94 94 94 F F F
63932024-02-15T03:00:04.849Z 6 120-139 57 57 57 103 103 103 F F F
63942024-02-15T03:00:04.849Z 7 140-159 57 57 57 103 103 103 F F F
63952024-02-15T03:00:04.849Z 8 160-179 55 55 55 101 101 101 F F F
63962024-02-15T03:00:05.274Z 9 180-199 51 51 51 94 94 94 F F F
63972024-02-15T03:00:05.274Z 10 200-219 53 53 53 98 98 98 F F F
63982024-02-15T03:00:05.274Z 11 220-239 57 57 57 103 103 103 F F F
63992024-02-15T03:00:05.274Z 12 240-259 57 57 57 103 103 103 F F F
64002024-02-15T03:00:05.274Z 13 260-279 44 44 44 83 83 83 F F F
64012024-02-15T03:00:05.275Z 14 280-299 57 57 57 103 103 103 F F F
64022024-02-15T03:00:05.275Z 15 300-319 52 52 52 95 95 95 F F F
64032024-02-15T03:00:05.276Z 16 320-339 52 52 52 97 97 97 F F F
64042024-02-15T03:00:05.276Z 17 340-359 53 53 53 98 98 98 F F F
64052024-02-15T03:00:05.293Z 18 360-379 51 51 51 94 94 94 F F F
64062024-02-15T03:00:05.293Z 19 380-399 53 53 53 98 98 98 F F F
64072024-02-15T03:00:05.293Z 20 400-419 57 57 57 103 103 103 F F F
64082024-02-15T03:00:05.293Z 21 420-439 57 57 57 103 103 103 F F F
64092024-02-15T03:00:05.293Z 22 440-459 57 57 57 103 103 103 F F F
64102024-02-15T03:00:05.293Z 23 460-479 57 57 57 103 103 103 F F F
64112024-02-15T03:00:05.293Z 24 480-499 57 57 57 103 103 103 F F F
64122024-02-15T03:00:05.294Z 25 500-519 50 50 50 91 91 91 F F F
64132024-02-15T03:00:05.294Z 26 520-539 55 55 55 101 101 101 F F F
64142024-02-15T03:00:05.294Z 27 540-559 55 55 55 101 101 101 F F F
64152024-02-15T03:00:05.294Z 28 560-579 57 57 57 103 103 103 F F F
64162024-02-15T03:00:05.294Z 29 580-599 57 57 57 103 103 103 F F F
64172024-02-15T03:00:05.294ZMax gen: 57, Max flush: 103
64182024-02-15T03:00:05.294Z
64192024-02-15T03:00:05.294ZBegin loop 29
64202024-02-15T03:00:05.294ZDownstairs to restart: 0
64212024-02-15T03:00:05.294Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776223208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367}
64222024-02-15T03:00:05.294Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\"Crucible runtime is spawned
64232024-02-15T03:00:05.295Z,\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776591502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367}
64242024-02-15T03:00:05.295Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776637767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367}
64252024-02-15T03:00:05.295Z{"msg":"Crucible 630b7153-2b14-4ce3-92e4-9edbc74528bb has session id: cdcd3ce1-afaa-4d77-9622-5f272c2d19aa","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776663773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64262024-02-15T03:00:05.295Z{"msg":"Upstairs opts: Upstairs UUID: 630b7153-2b14-4ce3-92e4-9edbc74528bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776706688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64272024-02-15T03:00:05.295Z{"msg":"Crucible stats registered with UUID: 630b7153-2b14-4ce3-92e4-9edbc74528bb","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776727415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64282024-02-15T03:00:05.295Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776743963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367}
64292024-02-15T03:00:05.295Z{"msg":"630b7153-2b14-4ce3-92e4-9edbc74528bb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776759221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64302024-02-15T03:00:05.296Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776783408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64312024-02-15T03:00:05.296Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776815054Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64322024-02-15T03:00:05.296Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776872887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64332024-02-15T03:00:05.296Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776953687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64342024-02-15T03:00:05.296Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.776994222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64352024-02-15T03:00:05.296Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777028707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64362024-02-15T03:00:05.296Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777159031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64372024-02-15T03:00:05.296Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777234282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64382024-02-15T03:00:05.296Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777287535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64392024-02-15T03:00:05.296Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777309332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64402024-02-15T03:00:05.296Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777337749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64412024-02-15T03:00:05.296Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:03.777369035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64422024-02-15T03:00:05.296Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:03.777389742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64432024-02-15T03:00:05.297Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:03.777419749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64442024-02-15T03:00:05.297Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777436816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64452024-02-15T03:00:05.297Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777462093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64462024-02-15T03:00:05.297Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777496319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64472024-02-15T03:00:05.297Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777891249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64482024-02-15T03:00:05.297Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777930405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64492024-02-15T03:00:05.297Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777946603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64502024-02-15T03:00:05.297Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.777960731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64512024-02-15T03:00:05.297Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.778457568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64522024-02-15T03:00:05.297Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.778568374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64532024-02-15T03:00:05.297Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.778964995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64542024-02-15T03:00:05.297Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.77900709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64552024-02-15T03:00:05.297Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.779030667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64562024-02-15T03:00:05.297Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.779180788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64572024-02-15T03:00:05.298Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.779211104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64582024-02-15T03:00:05.298Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:03.779225772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64592024-02-15T03:00:05.845Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.778909795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64602024-02-15T03:00:05.845Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.779048568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64612024-02-15T03:00:05.905Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.779891752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64622024-02-15T03:00:05.906Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.779927658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64632024-02-15T03:00:05.906Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.780458901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64642024-02-15T03:00:05.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781124208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64652024-02-15T03:00:05.906Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781165283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64662024-02-15T03:00:05.906Z{"msg":"[0]R flush_numbers[0..12]: [103, 103, 103, 103, 99, 94, 103, 103, 101, 94, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781203758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64672024-02-15T03:00:05.906Z{"msg":"[0]R generation[0..12]: [57, 57, 57, 57, 54, 51, 57, 57, 55, 51, 53, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781231234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64682024-02-15T03:00:05.906Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781261691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64692024-02-15T03:00:05.907Z{"msg":"[1]R flush_numbers[0..12]: [103, 103, 103, 103, 99, 94, 103, 103, 101, 94, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781277618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64702024-02-15T03:00:05.907Z{"msg":"[1]R generation[0..12]: [57, 57, 57, 57, 54, 51, 57, 57, 55, 51, 53, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781291627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64712024-02-15T03:00:05.907Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781313154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64722024-02-15T03:00:05.907Z{"msg":"[2]R flush_numbers[0..12]: [103, 103, 103, 103, 99, 94, 103, 103, 101, 94, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781328002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64732024-02-15T03:00:05.907Z{"msg":"[2]R generation[0..12]: [57, 57, 57, 57, 54, 51, 57, 57, 55, 51, 53, 57]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
64742024-02-15T03:00:05.907Z2024-02-15T03:00:04.781361868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64752024-02-15T03:00:05.907Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781401123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64762024-02-15T03:00:05.907Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781441358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64772024-02-15T03:00:05.908Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781462305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64782024-02-15T03:00:05.908Z{"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781479083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64792024-02-15T03:00:05.908Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781511579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64802024-02-15T03:00:05.908Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781543015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64812024-02-15T03:00:05.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.78158489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64822024-02-15T03:00:05.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781613856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64832024-02-15T03:00:05.908Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781631014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64842024-02-15T03:00:05.908Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781675619Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64852024-02-15T03:00:05.908Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781720353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64862024-02-15T03:00:05.908Z{"msg":"630b7153-2b14-4ce3-92e4-9edbc74528bb is now active with session: cdcd3ce1-afaa-4d77-9622-5f272c2d19aa","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.78174466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64872024-02-15T03:00:05.909Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781760458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367,"session_id":"cdcd3ce1-afaa-4d77-9622-5f272c2d19aa"}
64882024-02-15T03:00:05.909Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:04.781793604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367}
64892024-02-15T03:00:05.909ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
64902024-02-15T03:00:05.909ZLoading write count information from file "/tmp/test_repair_verify.data"
64912024-02-15T03:00:05.909ZRun Repair workload
64922024-02-15T03:00:05.909Z01/30 Flush
64932024-02-15T03:00:05.909Z02/30 Write block 293 len 16384 data: 7 8 6 7
64942024-02-15T03:00:05.909Z03/30 Read block 527 len 24576
64952024-02-15T03:00:05.910Z04/30 Read block 122 len 8192
64962024-02-15T03:00:05.910Z05/30 Read block 345 len 28672
64972024-02-15T03:00:05.910Z06/30 Read block 122 len 16384
64982024-02-15T03:00:05.910Z07/30 Flush
64992024-02-15T03:00:05.910Z08/30 Write block 283 len 4096 data: 3
65002024-02-15T03:00:05.910Z09/30 Read block 78 len 40960
65012024-02-15T03:00:05.945Z10/30 Read block 321 len 36864
65022024-02-15T03:00:05.945Z11/30 Write block 41 len 20480 data: 4 5 4 7 7
65032024-02-15T03:00:05.945Z12/30 Read block 107 len 20480
65042024-02-15T03:00:05.945Z13/30 Flush
65052024-02-15T03:00:05.945Z14/30 Write block 165 len 16384 data: 5 6 6 5
65062024-02-15T03:00:05.945Z15/30 Write block 508 len 28672 data: 5 7 6 7 9 9 9
65072024-02-15T03:00:05.945Z16/30 Write block 18 len 8192 data: 4 3
65082024-02-15T03:00:05.945Z17/30 Read block 41 len 36864
65092024-02-15T03:00:05.945Z18/30 Write block 460 len 20480 data: 4 4 5 4 4
65102024-02-15T03:00:05.946Z19/30 Read block 77 len 40960
65112024-02-15T03:00:05.946Z20/30 Read block 243 len 8192
65122024-02-15T03:00:05.946Z21/30 Read block 26 len 4096
65132024-02-15T03:00:05.946Z22/30 Write block 74 len 24576 data: 4 6 6 5 4 4
65142024-02-15T03:00:05.946Z23/30 Read block 278 len 4096
65152024-02-15T03:00:05.946Z24/30 Read block 331 len 8192
65162024-02-15T03:00:05.946Z25/30 Read block 209 len 32768
65172024-02-15T03:00:05.946Z26/30 Read block 150 len 8192
65182024-02-15T03:00:05.946Z27/30 Read block 223 len 12288
65192024-02-15T03:00:05.947Z28/30 Read block 411 len 36864
65202024-02-15T03:00:05.947Z29/30 Write block 580 len 4096 data: 6
65212024-02-15T03:00:05.947Z30/30 Read block 341 len 16384
65222024-02-15T03:00:05.947Z----------------------------------------------------------------
65232024-02-15T03:00:05.947Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:23
65242024-02-15T03:00:05.947ZStates: Active Active Active
65252024-02-15T03:00:05.947ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65262024-02-15T03:00:05.947Z 8 Acked 1007 Write 1 Sent Done Done false
65272024-02-15T03:00:05.947Z 9 Acked 1008 Read 10 Sent Done Done false
65282024-02-15T03:00:05.947Z 10 Acked 1009 Read 9 Sent Done Done false
65292024-02-15T03:00:05.947Z 11 Acked 1010 Write 5 Sent Done Done false
65302024-02-15T03:00:05.947Z 12 Acked 1011 Read 5 Sent Done Done false
65312024-02-15T03:00:05.947Z 13 Acked 1012 Flush 0 Sent Done Done false
65322024-02-15T03:00:05.947Z 14 Acked 1013 Write 4 Sent Done Done false
65332024-02-15T03:00:05.947Z 15 Acked 1014 Write 7 Sent Done Done false
65342024-02-15T03:00:05.947Z 16 Acked 1015 Write 2 Sent Done Done false
65352024-02-15T03:00:05.947Z 17 Acked 1016 Read 9 Sent Done Done false
65362024-02-15T03:00:05.947Z 18 Acked 1017 Write 5 Sent Done Done false
65372024-02-15T03:00:05.947Z 19 Acked 1018 Read 10 Sent Done Done false
65382024-02-15T03:00:05.947Z 20 Acked 1019 Read 2 Sent Done Done false
65392024-02-15T03:00:05.948Z 21 Acked 1020 Read 1 Sent Done Done false
65402024-02-15T03:00:05.948Z 22 Acked 1021 Write 6 Sent Done Done false
65412024-02-15T03:00:05.948Z 23 Acked 1022 Read 1 Sent Done Done false
65422024-02-15T03:00:05.948Z 24 Acked 1023 Read 2 Sent Done Done false
65432024-02-15T03:00:05.948Z 25 Acked 1024 Read 8 Sent Done Done false
65442024-02-15T03:00:05.948Z 26 Acked 1025 Read 2 Sent Done Done false
65452024-02-15T03:00:05.948Z 27 Acked 1026 Read 3 Sent Done Done false
65462024-02-15T03:00:05.949Z 28 Acked 1027 Read 9 Sent Done Done false
65472024-02-15T03:00:05.949Z 29 Acked 1028 Write 1 Sent Done Done false
65482024-02-15T03:00:05.949Z 30 Acked 1029 Read 4 Sent Done Sent false
65492024-02-15T03:00:05.949Z STATES DS:0 DS:1 DS:2 TOTAL
65502024-02-15T03:00:05.949Z New 0 0 0 0
65512024-02-15T03:00:05.949Z Sent 23 0 1 24
65522024-02-15T03:00:05.949Z Done 0 23 22 45
65532024-02-15T03:00:05.949Z Skipped 0 0 0 0
65542024-02-15T03:00:05.949Z Error 0 0 0 0
65552024-02-15T03:00:05.949ZLast Flush: 1006 1012 1012
65562024-02-15T03:00:05.949ZDownstairs last five completed: 1006 1005 1004 1003 1002
65572024-02-15T03:00:05.949ZUpstairs last five completed: 30 29 28 27 26
65582024-02-15T03:00:05.949Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:04.810491018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1367}
65592024-02-15T03:00:05.950ZWrote out file "/tmp/test_repair_verify.data"
65602024-02-15T03:00:05.950Z
65612024-02-15T03:00:05.950ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
65622024-02-15T03:00:05.950ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65632024-02-15T03:00:05.950Z 0 000-019 57 57 57 103 103 103 F T T <---
65642024-02-15T03:00:05.950Z 1 020-039 57 57 57 103 103 103 F F F
65652024-02-15T03:00:05.950Z 2 040-059 57 58 58 103 106 106 F F F <---
65662024-02-15T03:00:05.950Z 3 060-079 57 57 57 103 103 103 F T T <---
65672024-02-15T03:00:05.950Z 4 080-099 54 54 54 99 99 99 F F F
65682024-02-15T03:00:05.950Z 5 100-119 51 51 51 94 94 94 F F F
65692024-02-15T03:00:05.950Z 6 120-139 57 57 57 103 103 103 F F F
65702024-02-15T03:00:05.950Z 7 140-159 57 57 57 103 103 103 F F F
65712024-02-15T03:00:05.950Z 8 160-179 55 55 55 101 101 101 F T T <---
65722024-02-15T03:00:05.950Z 9 180-199 51 51 51 94 94 94 F F F
65732024-02-15T03:00:05.950Z 10 200-219 53 53 53 98 98 98 F F F
65742024-02-15T03:00:05.951Z 11 220-239 57 57 57 103 103 103 F F F
65752024-02-15T03:00:05.951Z 12 240-259 57 57 57 103 103 103 F F F
65762024-02-15T03:00:05.951Z 13 260-279 44 44 44 83 83 83 F F F
65772024-02-15T03:00:05.951Z 14 280-299 58 58 58 105 106 106 F F F <---
65782024-02-15T03:00:05.951Z 15 300-319 52 52 52 95 95 95 F F F
65792024-02-15T03:00:05.951Z 16 320-339 52 52 52 97 97 97 F F F
65802024-02-15T03:00:05.951Z 17 340-359 53 53 53 98 98 98 F F F
65812024-02-15T03:00:05.952Z 18 360-379 51 51 51 94 94 94 F F F
65822024-02-15T03:00:05.952Z 19 380-399 53 53 53 98 98 98 F F F
65832024-02-15T03:00:05.952Z 20 400-419 57 57 57 103 103 103 F F F
65842024-02-15T03:00:05.952Z 21 420-439 57 57 57 103 103 103 F F F
65852024-02-15T03:00:05.952Z 22 440-459 57 57 57 103 103 103 F F F
65862024-02-15T03:00:05.952Z 23 460-479 57 57 57 103 103 103 F T T <---
65872024-02-15T03:00:05.952Z 24 480-499 57 57 57 103 103 103 F F F
65882024-02-15T03:00:05.952Z 25 500-519 50 50 50 91 91 91 F T T <---
65892024-02-15T03:00:05.952Z 26 520-539 55 55 55 101 101 101 F F F
65902024-02-15T03:00:05.952Z 27 540-559 55 55 55 101 101 101 F F F
65912024-02-15T03:00:05.952Z 28 560-579 57 57 57 103 103 103 F F F
65922024-02-15T03:00:05.952Z 29 580-599 57 57 57 103 103 103 F T T <---
65932024-02-15T03:00:05.952ZMax gen: 58, Max flush: 106
65942024-02-15T03:00:05.952ZError: Difference in extent metadata found!
65952024-02-15T03:00:05.952ZA Difference in extent metadata is expected here
65962024-02-15T03:00:05.952ZOn loop 29
65972024-02-15T03:00:05.952Z
65982024-02-15T03:00:05.952ZVerifying data now
65992024-02-15T03:00:10.377ZNo change after verify
66002024-02-15T03:00:10.377ZLoop: 29 Downstairs dump after verify (and repair):
66012024-02-15T03:00:10.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66022024-02-15T03:00:10.390Z 0 000-019 59 59 59 107 107 107 F F F
66032024-02-15T03:00:10.390Z 1 020-039 57 57 57 103 103 103 F F F
66042024-02-15T03:00:10.390Z 2 040-059 58 58 58 106 106 106 F F F
66052024-02-15T03:00:10.390Z 3 060-079 59 59 59 107 107 107 F F F
66062024-02-15T03:00:10.391Z 4 080-099 54 54 54 99 99 99 F F F
66072024-02-15T03:00:10.391Z 5 100-119 51 51 51 94 94 94 F F F
66082024-02-15T03:00:10.391Z 6 120-139 57 57 57 103 103 103 F F F
66092024-02-15T03:00:10.391Z 7 140-159 57 57 57 103 103 103 F F F
66102024-02-15T03:00:10.391Z 8 160-179 59 59 59 107 107 107 F F F
66112024-02-15T03:00:10.391Z 9 180-199 51 51 51 94 94 94 F F F
66122024-02-15T03:00:10.391Z 10 200-219 53 53 53 98 98 98 F F F
66132024-02-15T03:00:10.391Z 11 220-239 57 57 57 103 103 103 F F F
66142024-02-15T03:00:10.391Z 12 240-259 57 57 57 103 103 103 F F F
66152024-02-15T03:00:10.391Z 13 260-279 44 44 44 83 83 83 F F F
66162024-02-15T03:00:10.392Z 14 280-299 58 58 58 106 106 106 F F F
66172024-02-15T03:00:10.392Z 15 300-319 52 52 52 95 95 95 F F F
66182024-02-15T03:00:10.392Z 16 320-339 52 52 52 97 97 97 F F F
66192024-02-15T03:00:10.392Z 17 340-359 53 53 53 98 98 98 F F F
66202024-02-15T03:00:10.392Z 18 360-379 51 51 51 94 94 94 F F F
66212024-02-15T03:00:10.392Z 19 380-399 53 53 53 98 98 98 F F F
66222024-02-15T03:00:10.392Z 20 400-419 57 57 57 103 103 103 F F F
66232024-02-15T03:00:10.392Z 21 420-439 57 57 57 103 103 103 F F F
66242024-02-15T03:00:10.392Z 22 440-459 57 57 57 103 103 103 F F F
66252024-02-15T03:00:10.392Z 23 460-479 59 59 59 107 107 107 F F F
66262024-02-15T03:00:10.392Z 24 480-499 57 57 57 103 103 103 F F F
66272024-02-15T03:00:10.393Z 25 500-519 59 59 59 107 107 107 F F F
66282024-02-15T03:00:10.393Z 26 520-539 55 55 55 101 101 101 F F F
66292024-02-15T03:00:10.393Z 27 540-559 55 55 55 101 101 101 F F F
66302024-02-15T03:00:10.393Z 28 560-579 57 57 57 103 103 103 F F F
66312024-02-15T03:00:10.393Z 29 580-599 59 59 59 107 107 107 F F F
66322024-02-15T03:00:10.393ZMax gen: 59, Max flush: 107
66332024-02-15T03:00:10.393Z
66342024-02-15T03:00:10.393ZBegin loop 30
66352024-02-15T03:00:10.394ZDownstairs to restart: 0
66362024-02-15T03:00:10.539Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.334519911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375}
66372024-02-15T03:00:10.539Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.334806635Z"Crucible runtime is spawned
66382024-02-15T03:00:10.539Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375}
66392024-02-15T03:00:10.539Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.33485182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375}
66402024-02-15T03:00:10.539Z{"msg":"Crucible d5b65629-9c7a-4a02-b485-126770dbebc0 has session id: 25d77345-0520-4266-9f91-5e7532c1496b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.334888775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66412024-02-15T03:00:10.539Z{"msg":"Upstairs opts: Upstairs UUID: d5b65629-9c7a-4a02-b485-126770dbebc0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.334914292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66422024-02-15T03:00:10.540Z{"msg":"Crucible stats registered with UUID: d5b65629-9c7a-4a02-b485-126770dbebc0","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.334937739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66432024-02-15T03:00:10.540Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.334951137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375}
66442024-02-15T03:00:10.540Z{"msg":"d5b65629-9c7a-4a02-b485-126770dbebc0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335041086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66452024-02-15T03:00:10.540Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335067903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66462024-02-15T03:00:10.540Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.33508739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66472024-02-15T03:00:10.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335123406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66482024-02-15T03:00:10.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335152192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66492024-02-15T03:00:10.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335191097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66502024-02-15T03:00:10.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335219374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66512024-02-15T03:00:10.541Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335262738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66522024-02-15T03:00:10.541Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335297424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66532024-02-15T03:00:10.542Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335324981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66542024-02-15T03:00:10.542Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:09.335355906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66552024-02-15T03:00:10.542Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:09.335383913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66562024-02-15T03:00:10.542Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.33541028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66572024-02-15T03:00:10.542Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:09.335440516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66582024-02-15T03:00:10.542Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335476351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66592024-02-15T03:00:10.542Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335501648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66602024-02-15T03:00:10.542Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.335538144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66612024-02-15T03:00:10.543Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.33557073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66622024-02-15T03:00:10.543Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.336084535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66632024-02-15T03:00:10.543Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.336121291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66642024-02-15T03:00:10.543Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.336137019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66652024-02-15T03:00:10.543Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.336223168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66662024-02-15T03:00:10.543Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.336653864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66672024-02-15T03:00:10.543Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.336773039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66682024-02-15T03:00:10.543Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.337130444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66692024-02-15T03:00:10.543Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.337171459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66702024-02-15T03:00:10.543Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.337186637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66712024-02-15T03:00:10.544Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.337306222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66722024-02-15T03:00:10.544Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.337342218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66732024-02-15T03:00:10.544Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:09.337359375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66742024-02-15T03:00:11.403Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.335936777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66752024-02-15T03:00:11.408Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.336158459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66762024-02-15T03:00:11.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.337182311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66772024-02-15T03:00:11.409Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.337217526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66782024-02-15T03:00:11.409Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.337709235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66792024-02-15T03:00:11.409Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338341555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66802024-02-15T03:00:11.409Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.33838385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66812024-02-15T03:00:11.409Z{"msg":"[0]R flush_numbers[0..12]: [107, 103, 106, 107, 99, 94, 103, 103, 107, 94, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338403548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66822024-02-15T03:00:11.409Z{"msg":"[0]R generation[0..12]: [59, 57, 58, 59, 54, 51, 57, 57, 59, 51, 53, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338430444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66832024-02-15T03:00:11.410Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338473659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66842024-02-15T03:00:11.410Z{"msg":"[1]R flush_numbers[0..12]: [107, 103, 106, 107, 99, 94, 103, 103, 107, 94, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338516693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66852024-02-15T03:00:11.410Z{"msg":"[1]R generation[0..12]: [59, 57, 58, 59, 54, 51, 57, 57, 59, 51, 53, 57]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
66862024-02-15T03:00:11.410Z,"time":"2024-02-15T03:00:10.338559338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66872024-02-15T03:00:11.410Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338605022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66882024-02-15T03:00:11.410Z{"msg":"[2]R flush_numbers[0..12]: [107, 103, 106, 107, 99, 94, 103, 103, 107, 94, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338646137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66892024-02-15T03:00:11.410Z{"msg":"[2]R generation[0..12]: [59, 57, 58, 59, 54, 51, 57, 57, 59, 51, 53, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338687332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66902024-02-15T03:00:11.410Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338733016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66912024-02-15T03:00:11.410Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338751854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66922024-02-15T03:00:11.410Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.33878092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66932024-02-15T03:00:11.410Z{"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338795689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66942024-02-15T03:00:11.410Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338812516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66952024-02-15T03:00:11.410Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338842263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66962024-02-15T03:00:11.410Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338858581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66972024-02-15T03:00:11.410Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338887727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66982024-02-15T03:00:11.410Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338907635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
66992024-02-15T03:00:11.410Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338936151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
67002024-02-15T03:00:11.410Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338953109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
67012024-02-15T03:00:11.410Z{"msg":"d5b65629-9c7a-4a02-b485-126770dbebc0 is now active with session: 25d77345-0520-4266-9f91-5e7532c1496b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.338981815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
67022024-02-15T03:00:11.410Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.339000453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375,"session_id":"25d77345-0520-4266-9f91-5e7532c1496b"}
67032024-02-15T03:00:11.410Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:10.339028519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375}
67042024-02-15T03:00:11.411ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67052024-02-15T03:00:11.411ZLoading write count information from file "/tmp/test_repair_verify.data"
67062024-02-15T03:00:11.411ZRun Repair workload
67072024-02-15T03:00:11.411Z01/30 Write block 591 len 36864 data: 7 5 6 8 6 4 4 4 2
67082024-02-15T03:00:11.417Z02/30 Read block 352 len 20480
67092024-02-15T03:00:11.417Z03/30 Write block 573 len 16384 data: 6 6 8 9
67102024-02-15T03:00:11.417Z04/30 Write block 174 len 32768 data: 5 5 4 5 5 5 5 4
67112024-02-15T03:00:11.418Z05/30 Write block 433 len 32768 data: 3 3 5 7 8 7 6 7
67122024-02-15T03:00:11.418Z06/30 Read block 130 len 20480
67132024-02-15T03:00:11.418Z07/30 Write block 285 len 16384 data: 5 6 6 4
67142024-02-15T03:00:11.418Z08/30 Read block 87 len 16384
67152024-02-15T03:00:11.418Z09/30 Read block 456 len 24576
67162024-02-15T03:00:11.418Z10/30 Write block 285 len 40960 data: 6 7 7 5 4 8 7 7 8 9
67172024-02-15T03:00:11.418Z11/30 Read block 51 len 8192
67182024-02-15T03:00:11.440Z12/30 Read block 229 len 36864
67192024-02-15T03:00:11.440Z13/30 Read block 0 len 36864
67202024-02-15T03:00:11.440Z14/30 Write block 479 len 36864 data: 4 3 5 5 4 5 4 5 6
67212024-02-15T03:00:11.440Z15/30 Read block 333 len 12288
67222024-02-15T03:00:11.440Z16/30 Read block 215 len 36864
67232024-02-15T03:00:11.441Z17/30 Write block 279 len 12288 data: 3 3 3
67242024-02-15T03:00:11.441Z18/30 Read block 158 len 36864
67252024-02-15T03:00:11.441Z19/30 Read block 167 len 32768
67262024-02-15T03:00:11.441Z20/30 Read block 250 len 20480
67272024-02-15T03:00:11.441Z21/30 Read block 403 len 12288
67282024-02-15T03:00:11.441Z22/30 Read block 62 len 16384
67292024-02-15T03:00:11.441Z23/30 Read block 430 len 36864
67302024-02-15T03:00:11.441Z24/30 Read block 72 len 8192
67312024-02-15T03:00:11.441Z25/30 Write block 471 len 36864 data: 8 7 6 5 3 3 4 5 5
67322024-02-15T03:00:11.441Z26/30 Write block 459 len 12288 data: 2 5 5
67332024-02-15T03:00:11.441Z27/30 Write block 284 len 12288 data: 5 7 8
67342024-02-15T03:00:11.441Z28/30 Write block 387 len 40960 data: 4 4 5 5 6 7 6 6 7 5
67352024-02-15T03:00:11.441Z29/30 Read block 442 len 20480
67362024-02-15T03:00:11.442Z30/30 Read block 494 len 40960
67372024-02-15T03:00:11.442Z----------------------------------------------------------------
67382024-02-15T03:00:11.442Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
67392024-02-15T03:00:11.442ZStates: Active Active Active
67402024-02-15T03:00:11.442ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67412024-02-15T03:00:11.442Z 1 Acked 1000 Write 9 Done Done Done false
67422024-02-15T03:00:11.442Z 2 Acked 1001 Read 5 Done Done Done false
67432024-02-15T03:00:11.442Z 3 Acked 1002 Write 4 Done Done Done false
67442024-02-15T03:00:11.442Z 4 Acked 1003 Write 8 Done Done Done false
67452024-02-15T03:00:11.442Z 5 Acked 1004 Write 8 Sent Done Done false
67462024-02-15T03:00:11.442Z 6 Acked 1005 Read 5 Sent Done Done false
67472024-02-15T03:00:11.442Z 7 Acked 1006 Write 4 Sent Done Done false
67482024-02-15T03:00:11.442Z 8 Acked 1007 Read 4 Sent Done Done false
67492024-02-15T03:00:11.442Z 9 Acked 1008 Read 6 Sent Done Done false
67502024-02-15T03:00:11.442Z 10 Acked 1009 Write 10 Sent Done Done false
67512024-02-15T03:00:11.442Z 11 Acked 1010 Read 2 Sent Done Done false
67522024-02-15T03:00:11.442Z 12 Acked 1011 Read 9 Sent Done Done false
67532024-02-15T03:00:11.442Z 13 Acked 1012 Read 9 Sent Done Done false
67542024-02-15T03:00:11.442Z 14 Acked 1013 Write 9 Sent Done Done false
67552024-02-15T03:00:11.442Z 15 Acked 1014 Read 3 Sent Done Done false
67562024-02-15T03:00:11.442Z 16 Acked 1015 Read 9 Sent Done Done false
67572024-02-15T03:00:11.442Z 17 Acked 1016 Write 3 Sent Done Done false
67582024-02-15T03:00:11.442Z 18 Acked 1017 Read 9 Sent Done Done false
67592024-02-15T03:00:11.443Z 19 Acked 1018 Read 8 Sent Done Done false
67602024-02-15T03:00:11.443Z 20 Acked 1019 Read 5 Sent Done Done false
67612024-02-15T03:00:11.443Z 21 Acked 1020 Read 3 Sent Done Done false
67622024-02-15T03:00:11.443Z 22 Acked 1021 Read 4 Sent Done Done false
67632024-02-15T03:00:11.443Z 23 Acked 1022 Read 9 Sent Done Done false
67642024-02-15T03:00:11.443Z 24 Acked 1023 Read 2 Sent Done Done false
67652024-02-15T03:00:11.443Z 25 Acked 1024 Write 9 Sent Done Done false
67662024-02-15T03:00:11.443Z 26 Acked 1025 Write 3 Sent Done Done false
67672024-02-15T03:00:11.443Z 27 Acked 1026 Write 3 Sent Done Done false
67682024-02-15T03:00:11.443Z 28 Acked 1027 Write 10 Sent Done Done false
67692024-02-15T03:00:11.443Z 29 Acked 1028 Read 5 Sent Done Done false
67702024-02-15T03:00:11.443Z 30 Acked 1029 Read 10 Sent Sent Done false
67712024-02-15T03:00:11.443Z STATES DS:0 DS:1 DS:2 TOTAL
67722024-02-15T03:00:11.443Z New 0 0 0 0
67732024-02-15T03:00:11.443Z Sent 26 1 0 27
67742024-02-15T03:00:11.443Z Done 4 29 30 63
67752024-02-15T03:00:11.443Z Skipped 0 0 0 0
67762024-02-15T03:00:11.443Z Error 0 0 0 0
67772024-02-15T03:00:11.443ZLast Flush: 0 0 0
67782024-02-15T03:00:11.444ZDownstairs last five completed:
67792024-02-15T03:00:11.444ZUpstairs last five completed: 30 29 28 27 26
67802024-02-15T03:00:11.444Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:10.370718609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1375}
67812024-02-15T03:00:11.444ZWrote out file "/tmp/test_repair_verify.data"
67822024-02-15T03:00:11.458Z
67832024-02-15T03:00:11.458ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
67842024-02-15T03:00:11.466ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67852024-02-15T03:00:11.466Z 0 000-019 59 59 59 107 107 107 F F F
67862024-02-15T03:00:11.466Z 1 020-039 57 57 57 103 103 103 F F F
67872024-02-15T03:00:11.466Z 2 040-059 58 58 58 106 106 106 F F F
67882024-02-15T03:00:11.466Z 3 060-079 59 59 59 107 107 107 F F F
67892024-02-15T03:00:11.466Z 4 080-099 54 54 54 99 99 99 F F F
67902024-02-15T03:00:11.466Z 5 100-119 51 51 51 94 94 94 F F F
67912024-02-15T03:00:11.471Z 6 120-139 57 57 57 103 103 103 F F F
67922024-02-15T03:00:11.471Z 7 140-159 57 57 57 103 103 103 F F F
67932024-02-15T03:00:11.471Z 8 160-179 59 59 59 107 107 107 T T T
67942024-02-15T03:00:11.471Z 9 180-199 51 51 51 94 94 94 T T T
67952024-02-15T03:00:11.471Z 10 200-219 53 53 53 98 98 98 F F F
67962024-02-15T03:00:11.471Z 11 220-239 57 57 57 103 103 103 F F F
67972024-02-15T03:00:11.471Z 12 240-259 57 57 57 103 103 103 F F F
67982024-02-15T03:00:11.471Z 13 260-279 44 44 44 83 83 83 F T T <---
67992024-02-15T03:00:11.471Z 14 280-299 58 58 58 106 106 106 F T T <---
68002024-02-15T03:00:11.471Z 15 300-319 52 52 52 95 95 95 F F F
68012024-02-15T03:00:11.471Z 16 320-339 52 52 52 97 97 97 F F F
68022024-02-15T03:00:11.471Z 17 340-359 53 53 53 98 98 98 F F F
68032024-02-15T03:00:11.471Z 18 360-379 51 51 51 94 94 94 F F F
68042024-02-15T03:00:11.471Z 19 380-399 53 53 53 98 98 98 F T T <---
68052024-02-15T03:00:11.471Z 20 400-419 57 57 57 103 103 103 F F F
68062024-02-15T03:00:11.472Z 21 420-439 57 57 57 103 103 103 F T T <---
68072024-02-15T03:00:11.472Z 22 440-459 57 57 57 103 103 103 F T T <---
68082024-02-15T03:00:11.472Z 23 460-479 59 59 59 107 107 107 F T T <---
68092024-02-15T03:00:11.472Z 24 480-499 57 57 57 103 103 103 F T T <---
68102024-02-15T03:00:11.472Z 25 500-519 59 59 59 107 107 107 F F F
68112024-02-15T03:00:11.472Z 26 520-539 55 55 55 101 101 101 F F F
68122024-02-15T03:00:11.472Z 27 540-559 55 55 55 101 101 101 F F F
68132024-02-15T03:00:11.472Z 28 560-579 57 57 57 103 103 103 T T T
68142024-02-15T03:00:11.472Z 29 580-599 59 59 59 107 107 107 T T T
68152024-02-15T03:00:11.472ZMax gen: 59, Max flush: 107
68162024-02-15T03:00:11.472ZError: Difference in extent metadata found!
68172024-02-15T03:00:11.472ZA Difference in extent metadata is expected here
68182024-02-15T03:00:11.472ZOn loop 30
68192024-02-15T03:00:11.472Z
68202024-02-15T03:00:11.493ZVerifying data now
68212024-02-15T03:00:17.071ZNo change after verify
68222024-02-15T03:00:17.071ZLoop: 30 Downstairs dump after verify (and repair):
68232024-02-15T03:00:17.121ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68242024-02-15T03:00:17.122Z 0 000-019 59 59 59 107 107 107 F F F
68252024-02-15T03:00:17.122Z 1 020-039 57 57 57 103 103 103 F F F
68262024-02-15T03:00:17.122Z 2 040-059 58 58 58 106 106 106 F F F
68272024-02-15T03:00:17.122Z 3 060-079 59 59 59 107 107 107 F F F
68282024-02-15T03:00:17.122Z 4 080-099 54 54 54 99 99 99 F F F
68292024-02-15T03:00:17.122Z 5 100-119 51 51 51 94 94 94 F F F
68302024-02-15T03:00:17.122Z 6 120-139 57 57 57 103 103 103 F F F
68312024-02-15T03:00:17.122Z 7 140-159 57 57 57 103 103 103 F F F
68322024-02-15T03:00:17.122Z 8 160-179 60 60 60 108 108 108 F F F
68332024-02-15T03:00:17.122Z 9 180-199 60 60 60 108 108 108 F F F
68342024-02-15T03:00:17.122Z 10 200-219 53 53 53 98 98 98 F F F
68352024-02-15T03:00:17.123Z 11 220-239 57 57 57 103 103 103 F F F
68362024-02-15T03:00:17.123Z 12 240-259 57 57 57 103 103 103 F F F
68372024-02-15T03:00:17.123Z 13 260-279 60 60 60 108 108 108 F F F
68382024-02-15T03:00:17.123Z 14 280-299 60 60 60 108 108 108 F F F
68392024-02-15T03:00:17.123Z 15 300-319 52 52 52 95 95 95 F F F
68402024-02-15T03:00:17.123Z 16 320-339 52 52 52 97 97 97 F F F
68412024-02-15T03:00:17.123Z 17 340-359 53 53 53 98 98 98 F F F
68422024-02-15T03:00:17.123Z 18 360-379 51 51 51 94 94 94 F F F
68432024-02-15T03:00:17.123Z 19 380-399 60 60 60 108 108 108 F F F
68442024-02-15T03:00:17.123Z 20 400-419 57 57 57 103 103 103 F F F
68452024-02-15T03:00:17.123Z 21 420-439 60 60 60 108 108 108 F F F
68462024-02-15T03:00:17.123Z 22 440-459 60 60 60 108 108 108 F F F
68472024-02-15T03:00:17.123Z 23 460-479 60 60 60 108 108 108 F F F
68482024-02-15T03:00:17.123Z 24 480-499 60 60 60 108 108 108 F F F
68492024-02-15T03:00:17.124Z 25 500-519 59 59 59 107 107 107 F F F
68502024-02-15T03:00:17.124Z 26 520-539 55 55 55 101 101 101 F F F
68512024-02-15T03:00:17.124Z 27 540-559 55 55 55 101 101 101 F F F
68522024-02-15T03:00:17.124Z 28 560-579 60 60 60 108 108 108 F F F
68532024-02-15T03:00:17.124Z 29 580-599 60 60 60 108 108 108 F F F
68542024-02-15T03:00:17.124ZMax gen: 60, Max flush: 108
68552024-02-15T03:00:17.124Z
68562024-02-15T03:00:17.124ZBegin loop 31
68572024-02-15T03:00:17.124ZDownstairs to restart: 2
68582024-02-15T03:00:17.124Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.04353577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383}
68592024-02-15T03:00:17.124Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.043862569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383}
68602024-02-15T03:00:17.125Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.043888296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383}
68612024-02-15T03:00:17.125Z{"msg":"Crucible cf90f5ec-e8ab-47de-ad0c-ec111c666039 has session id: d0923f10-bce5-4484-851e-c19999952645","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.043928051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68622024-02-15T03:00:17.125Z{"msg":"Upstairs opts: Upstairs UUID: cf90f5ec-e8ab-47de-ad0c-ec111c666039, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","vCrucible runtime is spawned
68632024-02-15T03:00:17.125Z":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.043998982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68642024-02-15T03:00:17.125Z{"msg":"Crucible stats registered with UUID: cf90f5ec-e8ab-47de-ad0c-ec111c666039","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044033688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68652024-02-15T03:00:17.125Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044082462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383}
68662024-02-15T03:00:17.125Z{"msg":"cf90f5ec-e8ab-47de-ad0c-ec111c666039 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044231253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68672024-02-15T03:00:17.125Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044263949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68682024-02-15T03:00:17.125Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044292646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68692024-02-15T03:00:17.125Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044333481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68702024-02-15T03:00:17.125Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044364366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68712024-02-15T03:00:17.125Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044391093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68722024-02-15T03:00:17.125Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.04441427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68732024-02-15T03:00:17.125Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044438517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68742024-02-15T03:00:17.126Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044481652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68752024-02-15T03:00:17.126Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044508968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68762024-02-15T03:00:17.126Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044661399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68772024-02-15T03:00:17.126Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:16.044706064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68782024-02-15T03:00:17.126Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:16.04473548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68792024-02-15T03:00:17.126Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:16.044778295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68802024-02-15T03:00:17.126Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.04481495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68812024-02-15T03:00:17.126Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044867023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68822024-02-15T03:00:17.127Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044900579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68832024-02-15T03:00:17.127Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.044925276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68842024-02-15T03:00:17.127Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.045423144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68852024-02-15T03:00:17.127Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.045461399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68862024-02-15T03:00:17.127Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.045487726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68872024-02-15T03:00:17.127Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.045515442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68882024-02-15T03:00:17.127Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.046117437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68892024-02-15T03:00:17.127Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.046155682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68902024-02-15T03:00:17.127Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.046711012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68912024-02-15T03:00:17.127Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.046747248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68922024-02-15T03:00:17.127Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.046763056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68932024-02-15T03:00:17.127Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.046780524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68942024-02-15T03:00:17.127Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.046797562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68952024-02-15T03:00:17.128Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:16.04681267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68962024-02-15T03:00:19.452Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.046431421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68972024-02-15T03:00:19.452Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.046544216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68982024-02-15T03:00:20.018Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.047449483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
68992024-02-15T03:00:20.018Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.047496737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69002024-02-15T03:00:20.018Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048013142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69012024-02-15T03:00:20.018Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048694797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69022024-02-15T03:00:20.019Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048742061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69032024-02-15T03:00:20.019Z{"msg":"[0]R flush_numbers[0..12]: [107, 103, 106, 107, 99, 94, 103, 103, 108, 108, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048770477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69042024-02-15T03:00:20.019Z{"msg":"[0]R generation[0..12]: [59, 57, 58, 59, 54, 51, 57, 57, 60, 60, 53, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048798544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69052024-02-15T03:00:20.019Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048814582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69062024-02-15T03:00:20.019Z{"msg":"[1]R flush_numbers[0..12]: [107, 103, 106, 107, 99, 94, 103, 103, 108, 108, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.04883099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69072024-02-15T03:00:20.019Z{"msg":"[1]R generation[0..12]: [59, 57, 58, 59, 54, 51, 57, 57, 60, 60, 53, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048853397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69082024-02-15T03:00:20.019Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048868635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69092024-02-15T03:00:20.019Z{"msg":"[2]R flush_numbers[0..12]: [107, 103, 106, 107, 99, 94, 103, 103, 108, 108, 98, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048883163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69102024-02-15T03:00:20.020Z{"msg":"[2]R generation[0..12]: [59, 57, 58, 59, 54, 51, 57, 57, 60, 60, 53, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048903911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69112024-02-15T03:00:20.020Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048918559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69122024-02-15T03:00:20.020ZWait for a query_work_queue command to finish before sending IO
69132024-02-15T03:00:20.020Z{"msg":"Max found gen is 61","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048948175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69142024-02-15T03:00:20.020Z{"msg":"Generation requested: 62 >= found:61","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048963763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69152024-02-15T03:00:20.020Z{"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048977102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69162024-02-15T03:00:20.020Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.048997439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69172024-02-15T03:00:20.020Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.049011727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69182024-02-15T03:00:20.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.049032445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69192024-02-15T03:00:20.020Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.049046393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69202024-02-15T03:00:20.021Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.04906707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69212024-02-15T03:00:20.021Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.049080958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69222024-02-15T03:00:20.021Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.049093587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69232024-02-15T03:00:20.021Z{"msg":"cf90f5ec-e8ab-47de-ad0c-ec111c666039 is now active with session: d0923f10-bce5-4484-851e-c19999952645","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.049106665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69242024-02-15T03:00:20.021Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.049131642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383,"session_id":"d0923f10-bce5-4484-851e-c19999952645"}
69252024-02-15T03:00:20.021Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:17.04914555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383}
69262024-02-15T03:00:20.021ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69272024-02-15T03:00:20.021ZLoading write count information from file "/tmp/test_repair_verify.data"
69282024-02-15T03:00:20.021ZRun Repair workload
69292024-02-15T03:00:20.021Z01/30 Write block 6 len 40960 data: 6 7 9 9 10 6 4 3 3 2
69302024-02-15T03:00:20.021Z02/30 Read block 60 len 12288
69312024-02-15T03:00:20.021Z03/30 Read block 196 len 20480
69322024-02-15T03:00:20.021Z04/30 Write block 196 len 40960 data: 7 6 4 3 4 4 3 3 3 4
69332024-02-15T03:00:20.022Z05/30 Write block 371 len 32768 data: 5 5 5 5 3 5 5 6
69342024-02-15T03:00:20.022Z06/30 Read block 328 len 12288
69352024-02-15T03:00:20.022Z07/30 Write block 156 len 4096 data: 5
69362024-02-15T03:00:20.022Z08/30 Read block 403 len 40960
69372024-02-15T03:00:20.022Z09/30 Write block 571 len 28672 data: 5 4 7 7 9 10 9
69382024-02-15T03:00:20.022Z10/30 Flush
69392024-02-15T03:00:20.022Z11/30 Write block 413 len 32768 data: 6 6 5 5 3 3 4 4
69402024-02-15T03:00:20.023Z12/30 Flush
69412024-02-15T03:00:20.023Z13/30 Write block 354 len 36864 data: 6 5 6 6 6 6 5 5 6
69422024-02-15T03:00:20.023Z14/30 Flush
69432024-02-15T03:00:20.023Z15/30 Write block 574 len 36864 data: 8 10 11 10 8 6 7 5 6
69442024-02-15T03:00:20.023Z16/30 Read block 51 len 28672
69452024-02-15T03:00:20.023Z17/30 Flush
69462024-02-15T03:00:20.024Z18/30 Flush
69472024-02-15T03:00:20.024Z19/30 Write block 379 len 28672 data: 5 6 7 7 8 8 5
69482024-02-15T03:00:20.024Z20/30 Read block 209 len 8192
69492024-02-15T03:00:20.024Z21/30 Write block 381 len 8192 data: 8 8
69502024-02-15T03:00:20.024Z22/30 Flush
69512024-02-15T03:00:20.024Z23/30 Write block 183 len 28672 data: 5 6 7 7 5 5 8
69522024-02-15T03:00:20.024Z24/30 Read block 20 len 4096
69532024-02-15T03:00:20.024Z25/30 Read block 23 len 32768
69542024-02-15T03:00:20.025Z26/30 Write block 145 len 24576 data: 8 7 8 6 5 4
69552024-02-15T03:00:20.025Z27/30 Read block 570 len 28672
69562024-02-15T03:00:20.025Z28/30 Read block 396 len 4096
69572024-02-15T03:00:20.025Z29/30 Write block 270 len 32768 data: 5 5 6 5 3 4 4 6
69582024-02-15T03:00:20.025Z30/30 Write block 466 len 24576 data: 6 6 7 8 9 9
69592024-02-15T03:00:20.025Z----------------------------------------------------------------
69602024-02-15T03:00:20.027Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
69612024-02-15T03:00:20.027ZStates: Active Active Active
69622024-02-15T03:00:20.027ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69632024-02-15T03:00:20.027Z 1 Acked 1000 Write 10 Done Done Done false
69642024-02-15T03:00:20.027Z 2 Acked 1001 Read 3 Done Done Done false
69652024-02-15T03:00:20.027Z 3 Acked 1002 Read 5 Done Done Done false
69662024-02-15T03:00:20.028Z 4 Acked 1003 Write 10 Done Done Sent false
69672024-02-15T03:00:20.028Z 5 Acked 1004 Write 8 Done Done Sent false
69682024-02-15T03:00:20.028Z 6 Acked 1005 Read 3 Done Done Sent false
69692024-02-15T03:00:20.028Z 7 Acked 1006 Write 1 Done Done Sent false
69702024-02-15T03:00:20.028Z 8 Acked 1007 Read 10 Done Done Sent false
69712024-02-15T03:00:20.028Z 9 Acked 1008 Write 7 Done Done Sent false
69722024-02-15T03:00:20.028Z 10 Acked 1009 Flush 0 Done Done Sent false
69732024-02-15T03:00:20.028Z 11 Acked 1010 Write 8 Done Done Sent false
69742024-02-15T03:00:20.029Z 12 Acked 1011 Flush 0 Done Done Sent false
69752024-02-15T03:00:20.029Z 13 Acked 1012 Write 9 Done Done Sent false
69762024-02-15T03:00:20.029Z 14 Acked 1013 Flush 0 Done Done Sent false
69772024-02-15T03:00:20.029Z 15 Acked 1014 Write 9 Done Done Sent false
69782024-02-15T03:00:20.029Z 16 Acked 1015 Read 7 Done Done Sent false
69792024-02-15T03:00:20.029Z 17 Acked 1016 Flush 0 Done Done Sent false
69802024-02-15T03:00:20.029Z 18 Acked 1017 Flush 0 Done Done Sent false
69812024-02-15T03:00:20.029Z 19 Acked 1018 Write 7 Done Done Sent false
69822024-02-15T03:00:20.029Z 20 Acked 1019 Read 2 Done Done Sent false
69832024-02-15T03:00:20.030Z 21 Acked 1020 Write 2 Done Done Sent false
69842024-02-15T03:00:20.030Z 22 Acked 1021 Flush 0 Done Done Sent false
69852024-02-15T03:00:20.030Z 23 Acked 1022 Write 7 Done Done Sent false
69862024-02-15T03:00:20.030Z 24 Acked 1023 Read 1 Done Done Sent false
69872024-02-15T03:00:20.030Z 25 Acked 1024 Read 8 Done Done Sent false
69882024-02-15T03:00:20.030Z 26 Acked 1025 Write 6 Done Done Sent false
69892024-02-15T03:00:20.030Z 27 Acked 1026 Read 7 Done Done Sent false
69902024-02-15T03:00:20.030Z 28 Acked 1027 Read 1 Done Done Sent false
69912024-02-15T03:00:20.031Z 29 Acked 1028 Write 8 Done Done Sent false
69922024-02-15T03:00:20.031Z 30 Acked 1029 Write 6 Sent Sent Sent false
69932024-02-15T03:00:20.031Z STATES DS:0 DS:1 DS:2 TOTAL
69942024-02-15T03:00:20.031Z New 0 0 0 0
69952024-02-15T03:00:20.031Z Sent 1 1 27 29
69962024-02-15T03:00:20.031Z Done 29 29 3 61
69972024-02-15T03:00:20.031Z Skipped 0 0 0 0
69982024-02-15T03:00:20.050Z Error 0 0 0 0
69992024-02-15T03:00:20.051ZLast Flush: 1021 1021 0
70002024-02-15T03:00:20.051ZDownstairs last five completed:
70012024-02-15T03:00:20.051ZUpstairs last five completed: 30 29 28 27 26
70022024-02-15T03:00:20.051Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:17.086746559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1383}
70032024-02-15T03:00:20.051ZWrote out file "/tmp/test_repair_verify.data"
70042024-02-15T03:00:20.051Z
70052024-02-15T03:00:20.051ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70062024-02-15T03:00:20.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70072024-02-15T03:00:20.051Z 0 000-019 62 62 59 109 109 107 F F T <---
70082024-02-15T03:00:20.051Z 1 020-039 57 57 57 103 103 103 F F F
70092024-02-15T03:00:20.051Z 2 040-059 58 58 58 106 106 106 F F F
70102024-02-15T03:00:20.051Z 3 060-079 59 59 59 107 107 107 F F F
70112024-02-15T03:00:20.051Z 4 080-099 54 54 54 99 99 99 F F F
70122024-02-15T03:00:20.051Z 5 100-119 51 51 51 94 94 94 F F F
70132024-02-15T03:00:20.051Z 6 120-139 57 57 57 103 103 103 F F F
70142024-02-15T03:00:20.051Z 7 140-159 62 62 57 109 109 103 T T F <---
70152024-02-15T03:00:20.051Z 8 160-179 60 60 60 108 108 108 F F F
70162024-02-15T03:00:20.051Z 9 180-199 62 62 60 109 109 108 T T F <---
70172024-02-15T03:00:20.051Z 10 200-219 62 62 53 109 109 98 F F F <---
70182024-02-15T03:00:20.051Z 11 220-239 57 57 57 103 103 103 F F F
70192024-02-15T03:00:20.051Z 12 240-259 57 57 57 103 103 103 F F F
70202024-02-15T03:00:20.052Z 13 260-279 60 60 60 108 108 108 T T F <---
70212024-02-15T03:00:20.052Z 14 280-299 60 60 60 108 108 108 F F F
70222024-02-15T03:00:20.052Z 15 300-319 52 52 52 95 95 95 F F F
70232024-02-15T03:00:20.052Z 16 320-339 52 52 52 97 97 97 F F F
70242024-02-15T03:00:20.052Z 17 340-359 62 62 53 111 111 98 F F F <---
70252024-02-15T03:00:20.052Z 18 360-379 62 62 51 114 114 94 F F F <---
70262024-02-15T03:00:20.052Z 19 380-399 62 62 60 114 114 108 F F F <---
70272024-02-15T03:00:20.052Z 20 400-419 62 62 57 110 110 103 F F F <---
70282024-02-15T03:00:20.052Z 21 420-439 62 62 60 110 110 108 F F F <---
70292024-02-15T03:00:20.052Z 22 440-459 60 60 60 108 108 108 F F F
70302024-02-15T03:00:20.052Z 23 460-479 60 60 60 108 108 108 T T F <---
70312024-02-15T03:00:20.052Z 24 480-499 60 60 60 108 108 108 F F F
70322024-02-15T03:00:20.052Z 25 500-519 59 59 59 107 107 107 F F F
70332024-02-15T03:00:20.052Z 26 520-539 55 55 55 101 101 101 F F F
70342024-02-15T03:00:20.052Z 27 540-559 55 55 55 101 101 101 F F F
70352024-02-15T03:00:20.052Z 28 560-579 62 62 60 112 112 108 F F F <---
70362024-02-15T03:00:20.052Z 29 580-599 62 62 60 112 112 108 F F F <---
70372024-02-15T03:00:20.052ZMax gen: 62, Max flush: 114
70382024-02-15T03:00:20.052ZError: Difference in extent metadata found!
70392024-02-15T03:00:20.052ZA Difference in extent metadata is expected here
70402024-02-15T03:00:20.053ZOn loop 31
70412024-02-15T03:00:20.053Z
70422024-02-15T03:00:20.053ZVerifying data now
70432024-02-15T03:00:27.252ZNo change after verify
70442024-02-15T03:00:27.252ZLoop: 31 Downstairs dump after verify (and repair):
70452024-02-15T03:00:27.591ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70462024-02-15T03:00:27.591Z 0 000-019 62 62 62 109 109 109 F F F
70472024-02-15T03:00:27.592Z 1 020-039 57 57 57 103 103 103 F F F
70482024-02-15T03:00:27.592Z 2 040-059 58 58 58 106 106 106 F F F
70492024-02-15T03:00:27.592Z 3 060-079 59 59 59 107 107 107 F F F
70502024-02-15T03:00:27.592Z 4 080-099 54 54 54 99 99 99 F F F
70512024-02-15T03:00:27.592Z 5 100-119 51 51 51 94 94 94 F F F
70522024-02-15T03:00:27.592Z 6 120-139 57 57 57 103 103 103 F F F
70532024-02-15T03:00:27.592Z 7 140-159 63 63 63 115 115 115 F F F
70542024-02-15T03:00:27.593Z 8 160-179 60 60 60 108 108 108 F F F
70552024-02-15T03:00:27.593Z 9 180-199 63 63 63 115 115 115 F F F
70562024-02-15T03:00:27.593Z 10 200-219 62 62 62 109 109 109 F F F
70572024-02-15T03:00:27.593Z 11 220-239 57 57 57 103 103 103 F F F
70582024-02-15T03:00:27.593Z 12 240-259 57 57 57 103 103 103 F F F
70592024-02-15T03:00:27.593Z 13 260-279 63 63 63 115 115 115 F F F
70602024-02-15T03:00:27.593Z 14 280-299 60 60 60 108 108 108 F F F
70612024-02-15T03:00:27.594Z 15 300-319 52 52 52 95 95 95 F F F
70622024-02-15T03:00:27.594Z 16 320-339 52 52 52 97 97 97 F F F
70632024-02-15T03:00:27.594Z 17 340-359 62 62 62 111 111 111 F F F
70642024-02-15T03:00:27.594Z 18 360-379 62 62 62 114 114 114 F F F
70652024-02-15T03:00:27.595Z 19 380-399 62 62 62 114 114 114 F F F
70662024-02-15T03:00:27.595Z 20 400-419 62 62 62 110 110 110 F F F
70672024-02-15T03:00:27.595Z 21 420-439 62 62 62 110 110 110 F F F
70682024-02-15T03:00:27.595Z 22 440-459 60 60 60 108 108 108 F F F
70692024-02-15T03:00:27.595Z 23 460-479 63 63 63 115 115 115 F F F
70702024-02-15T03:00:27.595Z 24 480-499 60 60 60 108 108 108 F F F
70712024-02-15T03:00:27.595Z 25 500-519 59 59 59 107 107 107 F F F
70722024-02-15T03:00:27.595Z 26 520-539 55 55 55 101 101 101 F F F
70732024-02-15T03:00:27.596Z 27 540-559 55 55 55 101 101 101 F F F
70742024-02-15T03:00:27.596Z 28 560-579 62 62 62 112 112 112 F F F
70752024-02-15T03:00:27.596Z 29 580-599 62 62 62 112 112 112 F F F
70762024-02-15T03:00:27.596ZMax gen: 63, Max flush: 115
70772024-02-15T03:00:27.596Z
70782024-02-15T03:00:27.596ZBegin loop 32
70792024-02-15T03:00:27.596ZDownstairs to restart: 2
70802024-02-15T03:00:27.597Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.815457786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391}
70812024-02-15T03:00:27.597Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.815791524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391}
70822024-02-15T03:00:27.597ZCrucible runtime is spawned
70832024-02-15T03:00:27.597Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.815839978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391}
70842024-02-15T03:00:27.597Z{"msg":"Crucible ada49a63-088e-4e58-925d-8fc74934cce6 has session id: e3034093-8011-4f9f-9ac5-6b621c22ef8d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.815872864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70852024-02-15T03:00:27.597Z{"msg":"Upstairs opts: Upstairs UUID: ada49a63-088e-4e58-925d-8fc74934cce6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.815909609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70862024-02-15T03:00:27.597Z{"msg":"Crucible stats registered with UUID: ada49a63-088e-4e58-925d-8fc74934cce6","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.815941995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70872024-02-15T03:00:27.597Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.815958903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391}
70882024-02-15T03:00:27.597Z{"msg":"ada49a63-088e-4e58-925d-8fc74934cce6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.8159866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70892024-02-15T03:00:27.598Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816015086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70902024-02-15T03:00:27.598Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816042543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70912024-02-15T03:00:27.598Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816059881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70922024-02-15T03:00:27.598Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816097776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70932024-02-15T03:00:27.598Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816160238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70942024-02-15T03:00:27.598Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816195904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70952024-02-15T03:00:27.598Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.81622724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70962024-02-15T03:00:27.598Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816287762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70972024-02-15T03:00:27.598Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816340795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70982024-02-15T03:00:27.599Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816430534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
70992024-02-15T03:00:27.599Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:21.816491576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71002024-02-15T03:00:27.599Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:21.816522712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71012024-02-15T03:00:27.599Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816552999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71022024-02-15T03:00:27.599Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:21.816660185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71032024-02-15T03:00:27.599Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816687572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71042024-02-15T03:00:27.599Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.81670512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71052024-02-15T03:00:27.599Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.816966737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71062024-02-15T03:00:27.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.817287317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71072024-02-15T03:00:27.600Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.817323062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71082024-02-15T03:00:27.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.817443867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71092024-02-15T03:00:27.600Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.817517128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71102024-02-15T03:00:27.600Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.817838588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71112024-02-15T03:00:27.600Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.817973771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71122024-02-15T03:00:27.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.818423105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71132024-02-15T03:00:27.600Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.818463379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71142024-02-15T03:00:27.601Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.818487956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71152024-02-15T03:00:27.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.818519582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71162024-02-15T03:00:27.601Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.818552558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71172024-02-15T03:00:27.602Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:21.818569086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71182024-02-15T03:00:27.602Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.817245655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71192024-02-15T03:00:27.602Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.817426402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71202024-02-15T03:00:27.602Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.818329969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71212024-02-15T03:00:27.602Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.818368484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71222024-02-15T03:00:27.602Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.818872941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71232024-02-15T03:00:27.602Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.8194369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71242024-02-15T03:00:27.603Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819467846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71252024-02-15T03:00:27.603Z{"msg":"[0]R flush_numbers[0..12]: [109, 103, 106, 107, 99, 94, 103, 115, 108, 115, 109, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819484604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71262024-02-15T03:00:27.603Z{"msg":"[0]R generation[0..12]: [62, 57, 58, 59, 54, 51, 57, 63, 60, 63, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819499193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71272024-02-15T03:00:27.603Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819539847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71282024-02-15T03:00:27.603Z{"msg":"[1]R flush_numbers[0..12]: [109, 103, 106, 107, 99, 94, 103, 115, 108, 115, 109, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819562725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71292024-02-15T03:00:27.604Z{"msg":"[1]R generation[0..12]: [62, 57, 58, 59, 54, 51, 57, 63, 60, 63, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819577003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71302024-02-15T03:00:27.604Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819591611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71312024-02-15T03:00:27.604Z{"msg":"[2]R flush_numbers[0..12]: [109, 103, 106, 107, 99, 94, 103, 115, 108, 115, 109, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819613338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71322024-02-15T03:00:27.604Z{"msg":"[2]R generation[0..12]: [62, 57, 58, 59, 54, 51, 57, 63, 60, 63, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819627646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71332024-02-15T03:00:27.604Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819649034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}Wait for a query_work_queue command to finish before sending IO
71342024-02-15T03:00:27.605Z
71352024-02-15T03:00:27.605Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819666781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71362024-02-15T03:00:27.605Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819687969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71372024-02-15T03:00:27.605Z{"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819701377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71382024-02-15T03:00:27.605Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819727444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71392024-02-15T03:00:27.606Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819742252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71402024-02-15T03:00:27.606Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819754481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71412024-02-15T03:00:27.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819783717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71422024-02-15T03:00:27.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819805754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71432024-02-15T03:00:27.607Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819826501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71442024-02-15T03:00:27.607Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.81984185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71452024-02-15T03:00:28.050Z{"msg":"ada49a63-088e-4e58-925d-8fc74934cce6 is now active with session: e3034093-8011-4f9f-9ac5-6b621c22ef8d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819859877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71462024-02-15T03:00:28.050Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819872216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391,"session_id":"e3034093-8011-4f9f-9ac5-6b621c22ef8d"}
71472024-02-15T03:00:28.051Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:22.819898072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391}
71482024-02-15T03:00:28.051ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71492024-02-15T03:00:28.051ZLoading write count information from file "/tmp/test_repair_verify.data"
71502024-02-15T03:00:28.051ZRun Repair workload
71512024-02-15T03:00:28.051Z01/30 Write block 279 len 12288 data: 4 4 4
71522024-02-15T03:00:28.051Z02/30 Read block 28 len 36864
71532024-02-15T03:00:28.051Z03/30 Read block 580 len 32768
71542024-02-15T03:00:28.051Z04/30 Write block 337 len 40960 data: 8 8 5 3 3 3 3 4 5 6
71552024-02-15T03:00:28.051Z05/30 Read block 15 len 20480
71562024-02-15T03:00:28.051Z06/30 Flush
71572024-02-15T03:00:28.051Z07/30 Flush
71582024-02-15T03:00:28.052Z08/30 Write block 208 len 36864 data: 6 4 4 5 6 6 7 6 5
71592024-02-15T03:00:28.052Z09/30 Read block 330 len 28672
71602024-02-15T03:00:28.052Z10/30 Write block 156 len 28672 data: 6 6 5 5 4 4 6
71612024-02-15T03:00:28.052Z11/30 Read block 125 len 12288
71622024-02-15T03:00:28.052Z12/30 Write block 328 len 28672 data: 7 7 7 9 10 10 11
71632024-02-15T03:00:28.052Z13/30 Flush
71642024-02-15T03:00:28.052Z14/30 Write block 382 len 36864 data: 9 9 9 6 4 5 5 6 6
71652024-02-15T03:00:28.052Z15/30 Read block 468 len 8192
71662024-02-15T03:00:28.052Z16/30 Read block 243 len 12288
71672024-02-15T03:00:28.052Z17/30 Read block 309 len 28672
71682024-02-15T03:00:28.052Z18/30 Write block 136 len 12288 data: 7 6 6
71692024-02-15T03:00:28.052Z19/30 Read block 391 len 28672
71702024-02-15T03:00:28.052Z20/30 Write block 161 len 8192 data: 5 7
71712024-02-15T03:00:28.053Z21/30 Write block 257 len 20480 data: 6 5 6 6 6
71722024-02-15T03:00:28.053Z22/30 Write block 317 len 12288 data: 5 5 4
71732024-02-15T03:00:28.053Z23/30 Write block 141 len 8192 data: 8 8
71742024-02-15T03:00:28.053Z24/30 Read block 387 len 36864
71752024-02-15T03:00:28.053Z25/30 Flush
71762024-02-15T03:00:28.053Z26/30 Write block 581 len 24576 data: 6 7 5 5 4 5
71772024-02-15T03:00:28.053Z27/30 Write block 373 len 12288 data: 6 6 4
71782024-02-15T03:00:28.053Z28/30 Write block 246 len 28672 data: 7 5 5 6 6 5 6
71792024-02-15T03:00:28.053Z29/30 Read block 251 len 12288
71802024-02-15T03:00:28.053Z30/30 Write block 47 len 24576 data: 9 8 7 4 2 4
71812024-02-15T03:00:28.053Z----------------------------------------------------------------
71822024-02-15T03:00:28.054Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
71832024-02-15T03:00:28.054ZStates: Active Active Active
71842024-02-15T03:00:28.054ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71852024-02-15T03:00:28.054Z 1 Acked 1000 Write 3 Done Done Done false
71862024-02-15T03:00:28.054Z 2 Acked 1001 Read 9 Done Done Done false
71872024-02-15T03:00:28.054Z 3 Acked 1002 Read 8 Done Done Sent false
71882024-02-15T03:00:28.054Z 4 Acked 1003 Write 10 Done Done Sent false
71892024-02-15T03:00:28.054Z 5 Acked 1004 Read 5 Done Done Sent false
71902024-02-15T03:00:28.054Z 6 Acked 1005 Flush 0 Done Done Sent false
71912024-02-15T03:00:28.054Z 7 Acked 1006 Flush 0 Done Done Sent false
71922024-02-15T03:00:28.054Z 8 Acked 1007 Write 9 Done Done Sent false
71932024-02-15T03:00:28.054Z 9 Acked 1008 Read 7 Done Done Sent false
71942024-02-15T03:00:28.054Z 10 Acked 1009 Write 7 Done Done Sent false
71952024-02-15T03:00:28.055Z 11 Acked 1010 Read 3 Done Done Sent false
71962024-02-15T03:00:28.055Z 12 Acked 1011 Write 7 Done Done Sent false
71972024-02-15T03:00:28.055Z 13 Acked 1012 Flush 0 Done Done Sent false
71982024-02-15T03:00:28.055Z 14 Acked 1013 Write 9 Done Done Sent false
71992024-02-15T03:00:28.055Z 15 Acked 1014 Read 2 Done Done Sent false
72002024-02-15T03:00:28.055Z 16 Acked 1015 Read 3 Done Done Sent false
72012024-02-15T03:00:28.055Z 17 Acked 1016 Read 7 Done Done Sent false
72022024-02-15T03:00:28.055Z 18 Acked 1017 Write 3 Done Done Sent false
72032024-02-15T03:00:28.055Z 19 Acked 1018 Read 7 Done Done Sent false
72042024-02-15T03:00:28.055Z 20 Acked 1019 Write 2 Done Done Sent false
72052024-02-15T03:00:28.055Z 21 Acked 1020 Write 5 Done Done Sent false
72062024-02-15T03:00:28.055Z 22 Acked 1021 Write 3 Done Done Sent false
72072024-02-15T03:00:28.055Z 23 Acked 1022 Write 2 Done Done Sent false
72082024-02-15T03:00:28.055Z 24 Acked 1023 Read 9 Done Done Sent false
72092024-02-15T03:00:28.055Z 25 Acked 1024 Flush 0 Done Done Sent false
72102024-02-15T03:00:28.056Z 26 Acked 1025 Write 6 Done Done Sent false
72112024-02-15T03:00:28.056Z 27 Acked 1026 Write 3 Done Done Sent false
72122024-02-15T03:00:28.056Z 28 Acked 1027 Write 7 Done Done Sent false
72132024-02-15T03:00:28.056Z 29 Acked 1028 Read 3 Done Done Sent false
72142024-02-15T03:00:28.056Z 30 Acked 1029 Write 6 Sent Sent Sent false
72152024-02-15T03:00:28.056Z STATES DS:0 DS:1 DS:2 TOTAL
72162024-02-15T03:00:28.056Z New 0 0 0 0
72172024-02-15T03:00:28.056Z Sent 1 1 28 30
72182024-02-15T03:00:28.056Z Done 29 29 2 60
72192024-02-15T03:00:28.056Z Skipped 0 0 0 0
72202024-02-15T03:00:28.056Z Error 0 0 0 0
72212024-02-15T03:00:28.056ZLast Flush: 1024 1024 0
72222024-02-15T03:00:28.056ZDownstairs last five completed:
72232024-02-15T03:00:28.056ZUpstairs last five completed: 30 29 28 27 26
72242024-02-15T03:00:28.056Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:22.852999225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1391}
72252024-02-15T03:00:28.057ZWrote out file "/tmp/test_repair_verify.data"
72262024-02-15T03:00:28.057Z
72272024-02-15T03:00:28.057ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72282024-02-15T03:00:28.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72292024-02-15T03:00:28.057Z 0 000-019 62 62 62 109 109 109 F F F
72302024-02-15T03:00:28.057Z 1 020-039 57 57 57 103 103 103 F F F
72312024-02-15T03:00:28.057Z 2 040-059 58 58 58 106 106 106 T T F <---
72322024-02-15T03:00:28.057Z 3 060-079 59 59 59 107 107 107 F F F
72332024-02-15T03:00:28.057Z 4 080-099 54 54 54 99 99 99 F F F
72342024-02-15T03:00:28.057Z 5 100-119 51 51 51 94 94 94 F F F
72352024-02-15T03:00:28.057Z 6 120-139 64 64 57 119 119 103 F F F <---
72362024-02-15T03:00:28.057Z 7 140-159 64 64 63 119 119 115 F F F <---
72372024-02-15T03:00:28.057Z 8 160-179 64 64 60 119 119 108 F F F <---
72382024-02-15T03:00:28.057Z 9 180-199 63 63 63 115 115 115 F F F
72392024-02-15T03:00:28.057Z 10 200-219 64 64 62 118 118 109 F F F <---
72402024-02-15T03:00:28.057Z 11 220-239 57 57 57 103 103 103 F F F
72412024-02-15T03:00:28.058Z 12 240-259 64 64 57 119 119 103 T T F <---
72422024-02-15T03:00:28.058Z 13 260-279 64 64 63 119 119 115 F F T <---
72432024-02-15T03:00:28.058Z 14 280-299 64 64 60 116 116 108 F F T <---
72442024-02-15T03:00:28.058Z 15 300-319 64 64 52 119 119 95 F F F <---
72452024-02-15T03:00:28.066Z 16 320-339 64 64 52 118 118 97 F F F <---
72462024-02-15T03:00:28.066Z 17 340-359 64 64 62 116 116 111 F F F <---
72472024-02-15T03:00:28.066Z 18 360-379 62 62 62 114 114 114 T T F <---
72482024-02-15T03:00:28.066Z 19 380-399 64 64 62 119 119 114 F F F <---
72492024-02-15T03:00:28.066Z 20 400-419 62 62 62 110 110 110 F F F
72502024-02-15T03:00:28.066Z 21 420-439 62 62 62 110 110 110 F F F
72512024-02-15T03:00:28.066Z 22 440-459 60 60 60 108 108 108 F F F
72522024-02-15T03:00:28.066Z 23 460-479 63 63 63 115 115 115 F F F
72532024-02-15T03:00:28.066Z 24 480-499 60 60 60 108 108 108 F F F
72542024-02-15T03:00:28.066Z 25 500-519 59 59 59 107 107 107 F F F
72552024-02-15T03:00:28.066Z 26 520-539 55 55 55 101 101 101 F F F
72562024-02-15T03:00:28.067Z 27 540-559 55 55 55 101 101 101 F F F
72572024-02-15T03:00:28.067Z 28 560-579 62 62 62 112 112 112 F F F
72582024-02-15T03:00:28.067Z 29 580-599 62 62 62 112 112 112 T T F <---
72592024-02-15T03:00:28.067ZMax gen: 64, Max flush: 119
72602024-02-15T03:00:28.067ZError: Difference in extent metadata found!
72612024-02-15T03:00:28.067ZA Difference in extent metadata is expected here
72622024-02-15T03:00:28.067ZOn loop 32
72632024-02-15T03:00:28.067Z
72642024-02-15T03:00:28.067ZVerifying data now
72652024-02-15T03:00:29.661ZNo change after verify
72662024-02-15T03:00:29.662ZLoop: 32 Downstairs dump after verify (and repair):
72672024-02-15T03:00:29.674ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72682024-02-15T03:00:29.674Z 0 000-019 62 62 62 109 109 109 F F F
72692024-02-15T03:00:29.674Z 1 020-039 57 57 57 103 103 103 F F F
72702024-02-15T03:00:29.674Z 2 040-059 65 65 65 120 120 120 F F F
72712024-02-15T03:00:29.674Z 3 060-079 59 59 59 107 107 107 F F F
72722024-02-15T03:00:29.674Z 4 080-099 54 54 54 99 99 99 F F F
72732024-02-15T03:00:29.680Z 5 100-119 51 51 51 94 94 94 F F F
72742024-02-15T03:00:29.680Z 6 120-139 64 64 64 119 119 119 F F F
72752024-02-15T03:00:29.680Z 7 140-159 64 64 64 119 119 119 F F F
72762024-02-15T03:00:29.680Z 8 160-179 64 64 64 119 119 119 F F F
72772024-02-15T03:00:29.680Z 9 180-199 63 63 63 115 115 115 F F F
72782024-02-15T03:00:29.680Z 10 200-219 64 64 64 118 118 118 F F F
72792024-02-15T03:00:29.680Z 11 220-239 57 57 57 103 103 103 F F F
72802024-02-15T03:00:29.680Z 12 240-259 65 65 65 120 120 120 F F F
72812024-02-15T03:00:29.680Z 13 260-279 64 64 64 119 119 119 F F F
72822024-02-15T03:00:29.680Z 14 280-299 64 64 64 116 116 116 F F F
72832024-02-15T03:00:29.680Z 15 300-319 64 64 64 119 119 119 F F F
72842024-02-15T03:00:29.680Z 16 320-339 64 64 64 118 118 118 F F F
72852024-02-15T03:00:29.680Z 17 340-359 64 64 64 116 116 116 F F F
72862024-02-15T03:00:29.681Z 18 360-379 65 65 65 120 120 120 F F F
72872024-02-15T03:00:29.681Z 19 380-399 64 64 64 119 119 119 F F F
72882024-02-15T03:00:29.681Z 20 400-419 62 62 62 110 110 110 F F F
72892024-02-15T03:00:29.681Z 21 420-439 62 62 62 110 110 110 F F F
72902024-02-15T03:00:29.681Z 22 440-459 60 60 60 108 108 108 F F F
72912024-02-15T03:00:29.681Z 23 460-479 63 63 63 115 115 115 F F F
72922024-02-15T03:00:29.681Z 24 480-499 60 60 60 108 108 108 F F F
72932024-02-15T03:00:29.681Z 25 500-519 59 59 59 107 107 107 F F F
72942024-02-15T03:00:29.681Z 26 520-539 55 55 55 101 101 101 F F F
72952024-02-15T03:00:29.681Z 27 540-559 55 55 55 101 101 101 F F F
72962024-02-15T03:00:29.681Z 28 560-579 62 62 62 112 112 112 F F F
72972024-02-15T03:00:29.681Z 29 580-599 65 65 65 120 120 120 F F F
72982024-02-15T03:00:29.681ZMax gen: 65, Max flush: 120
72992024-02-15T03:00:29.681Z
73002024-02-15T03:00:29.682ZBegin loop 33
73012024-02-15T03:00:29.682ZDownstairs to restart: 1
73022024-02-15T03:00:29.723Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624019213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399}
73032024-02-15T03:00:29.723Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624331614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399}
73042024-02-15T03:00:29.723Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624349002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399}
73052024-02-15T03:00:29.723Z{"msg":"Crucible 207287d6-17ea-485c-b811-542fd5851c18 has session id: 6f4b0a30-7d19-48b6-be4c-ee5bfb052e56","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624384578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73062024-02-15T03:00:29.723Z{"msg":"Upstairs opts: Upstairs UUID: 207287d6-17ea-485c-b811-542fd5851c18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624416724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73072024-02-15T03:00:29.723Z{"msg":"Crucible stats registered with UUID: 207287d6-17ea-485c-b811-542fd5851c18","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624442151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73082024-02-15T03:00:29.723ZCrucible runtime is spawned
73092024-02-15T03:00:29.723Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624615809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399}
73102024-02-15T03:00:29.723Z{"msg":"207287d6-17ea-485c-b811-542fd5851c18 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624742563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73112024-02-15T03:00:29.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624797916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73122024-02-15T03:00:29.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.62484807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73132024-02-15T03:00:29.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624904163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73142024-02-15T03:00:29.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624939638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73152024-02-15T03:00:29.724Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.624982523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73162024-02-15T03:00:29.724Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625013019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73172024-02-15T03:00:29.724Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.62516649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73182024-02-15T03:00:29.724Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625206125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73192024-02-15T03:00:29.724Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625222853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73202024-02-15T03:00:29.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625254419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73212024-02-15T03:00:29.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625278226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73222024-02-15T03:00:29.724Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:28.625308482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73232024-02-15T03:00:29.724Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:28.625343567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73242024-02-15T03:00:29.725Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:28.625373334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73252024-02-15T03:00:29.725Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625396341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73262024-02-15T03:00:29.725Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625469362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73272024-02-15T03:00:29.725Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.62556126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73282024-02-15T03:00:29.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625764295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73292024-02-15T03:00:29.725Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625813739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73302024-02-15T03:00:29.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625892899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73312024-02-15T03:00:29.725Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.625928704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73322024-02-15T03:00:29.725Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.626384057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73332024-02-15T03:00:29.725Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.626430641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73342024-02-15T03:00:29.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.626982022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73352024-02-15T03:00:29.725Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.627023557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73362024-02-15T03:00:29.725Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.627061552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73372024-02-15T03:00:29.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.627085619Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73382024-02-15T03:00:29.725Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.627116745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73392024-02-15T03:00:29.725Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:28.627142302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73402024-02-15T03:00:30.699Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.62662219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73412024-02-15T03:00:30.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.626753354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73422024-02-15T03:00:30.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.627619115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73432024-02-15T03:00:30.705Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.62766239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73442024-02-15T03:00:30.705Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.628193043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73452024-02-15T03:00:30.705Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.628881907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73462024-02-15T03:00:30.705Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.628931551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73472024-02-15T03:00:30.705Z{"msg":"[0]R flush_numbers[0..12]: [109, 103, 120, 107, 99, 94, 119, 119, 119, 115, 118, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.628965537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73482024-02-15T03:00:30.705Z{"msg":"[0]R generation[0..12]: [62, 57, 65, 59, 54, 51, 64, 64, 64, 63, 64, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629003392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73492024-02-15T03:00:30.706Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
73502024-02-15T03:00:30.706Z,"time":"2024-02-15T03:00:29.629036538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73512024-02-15T03:00:30.706Z{"msg":"[1]R flush_numbers[0..12]: [109, 103, 120, 107, 99, 94, 119, 119, 119, 115, 118, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629068564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73522024-02-15T03:00:30.706Z{"msg":"[1]R generation[0..12]: [62, 57, 65, 59, 54, 51, 64, 64, 64, 63, 64, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629108409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73532024-02-15T03:00:30.706Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629140995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73542024-02-15T03:00:30.707Z{"msg":"[2]R flush_numbers[0..12]: [109, 103, 120, 107, 99, 94, 119, 119, 119, 115, 118, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.62917906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73552024-02-15T03:00:30.707Z{"msg":"[2]R generation[0..12]: [62, 57, 65, 59, 54, 51, 64, 64, 64, 63, 64, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629209476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73562024-02-15T03:00:30.707Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629240652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73572024-02-15T03:00:30.707Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629267099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73582024-02-15T03:00:30.707Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629313773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73592024-02-15T03:00:30.707Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629343869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73602024-02-15T03:00:30.707Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629370866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73612024-02-15T03:00:30.708Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629398842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73622024-02-15T03:00:30.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629424779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73632024-02-15T03:00:30.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629463424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73642024-02-15T03:00:30.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.62949437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73652024-02-15T03:00:30.708Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629522857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73662024-02-15T03:00:30.708Z{"msg":"Set Downstairs and Upstairs active after reconciliation"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73672024-02-15T03:00:30.708Z,"v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629563702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73682024-02-15T03:00:30.708Z{"msg":"207287d6-17ea-485c-b811-542fd5851c18 is now active with session: 6f4b0a30-7d19-48b6-be4c-ee5bfb052e56","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629589688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73692024-02-15T03:00:30.709Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629612075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399,"session_id":"6f4b0a30-7d19-48b6-be4c-ee5bfb052e56"}
73702024-02-15T03:00:30.709Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:29.629634073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399}
73712024-02-15T03:00:30.709ZLoading write count information from file "/tmp/test_repair_verify.data"
73722024-02-15T03:00:30.709ZRun Repair workload
73732024-02-15T03:00:30.709Z01/30 Write block 311 len 24576 data: 8 7 8 6 5 5
73742024-02-15T03:00:30.709Z02/30 Read block 197 len 4096
73752024-02-15T03:00:30.710Z03/30 Write block 475 len 40960 data: 4 4 5 6 6 4 6 6 5 6
73762024-02-15T03:00:30.717Z04/30 Write block 553 len 8192 data: 4 4
73772024-02-15T03:00:30.717Z05/30 Read block 9 len 20480
73782024-02-15T03:00:30.717Z06/30 Read block 124 len 32768
73792024-02-15T03:00:30.718Z07/30 Write block 63 len 32768 data: 6 6 5 4 4 5 5 4
73802024-02-15T03:00:30.718Z08/30 Read block 553 len 16384
73812024-02-15T03:00:30.718Z09/30 Write block 155 len 16384 data: 3 7 7 6
73822024-02-15T03:00:30.718Z10/30 Read block 100 len 12288
73832024-02-15T03:00:30.718Z11/30 Read block 320 len 40960
73842024-02-15T03:00:30.718Z12/30 Write block 234 len 32768 data: 7 7 8 6 5 5 6 6
73852024-02-15T03:00:30.718Z13/30 Write block 532 len 24576 data: 6 4 4 5 5 6
73862024-02-15T03:00:30.718Z14/30 Write block 151 len 32768 data: 4 4 5 4 4 8 8 7
73872024-02-15T03:00:30.718Z15/30 Read block 532 len 36864
73882024-02-15T03:00:30.718Z16/30 Write block 480 len 4096 data: 5
73892024-02-15T03:00:30.718Z17/30 Write block 81 len 36864 data: 5 4 3 3 3 3 4 3 3
73902024-02-15T03:00:30.726Z18/30 Write block 153 len 40960 data: 6 5 5 9 9 8 6 5 6 8
73912024-02-15T03:00:30.726Z19/30 Write block 553 len 8192 data: 5 5
73922024-02-15T03:00:30.726Z20/30 Write block 270 len 32768 data: 6 6 7 6 4 5 5 7
73932024-02-15T03:00:30.727Z21/30 Flush
73942024-02-15T03:00:30.736Z22/30 Write block 383 len 20480 data: 10 10 7 5 6
73952024-02-15T03:00:30.736Z23/30 Read block 60 len 8192
73962024-02-15T03:00:30.737Z24/30 Write block 403 len 36864 data: 3 3 6 7 7 7 8 8 7
73972024-02-15T03:00:30.737Z25/30 Write block 220 len 28672 data: 9 9 9 7 7 7 9
73982024-02-15T03:00:30.737Z26/30 Write block 562 len 36864 data: 6 6 6 6 3 3 3 2 4
73992024-02-15T03:00:30.737Z27/30 Write block 423 len 24576 data: 4 5 5 6 6 5
74002024-02-15T03:00:30.737Z28/30 Read block 11 len 40960
74012024-02-15T03:00:30.737Z29/30 Read block 283 len 16384
74022024-02-15T03:00:30.738Z30/30 Read block 312 len 32768
74032024-02-15T03:00:30.759Z----------------------------------------------------------------
74042024-02-15T03:00:30.759Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
74052024-02-15T03:00:30.759ZStates: Active Active Active
74062024-02-15T03:00:30.759ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74072024-02-15T03:00:30.760Z 1 Acked 1000 Write 6 Done Done Done false
74082024-02-15T03:00:30.760Z 2 Acked 1001 Read 1 Done Done Done false
74092024-02-15T03:00:30.760Z 3 Acked 1002 Write 10 Done Sent Done false
74102024-02-15T03:00:30.760Z 4 Acked 1003 Write 2 Done Sent Done false
74112024-02-15T03:00:30.760Z 5 Acked 1004 Read 5 Done Sent Done false
74122024-02-15T03:00:30.760Z 6 Acked 1005 Read 8 Done Sent Done false
74132024-02-15T03:00:30.761Z 7 Acked 1006 Write 8 Done Sent Done false
74142024-02-15T03:00:30.761Z 8 Acked 1007 Read 4 Done Sent Done false
74152024-02-15T03:00:30.761Z 9 Acked 1008 Write 4 Done Sent Done false
74162024-02-15T03:00:30.761Z 10 Acked 1009 Read 3 Done Sent Done false
74172024-02-15T03:00:30.761Z 11 Acked 1010 Read 10 Done Sent Done false
74182024-02-15T03:00:30.761Z 12 Acked 1011 Write 8 Done Sent Done false
74192024-02-15T03:00:30.762Z 13 Acked 1012 Write 6 Done Sent Done false
74202024-02-15T03:00:30.762Z 14 Acked 1013 Write 8 Done Sent Done false
74212024-02-15T03:00:30.762Z 15 Acked 1014 Read 9 Done Sent Done false
74222024-02-15T03:00:30.764Z 16 Acked 1015 Write 1 Done Sent Done false
74232024-02-15T03:00:30.764Z 17 Acked 1016 Write 9 Done Sent Done false
74242024-02-15T03:00:30.764Z 18 Acked 1017 Write 10 Done Sent Done false
74252024-02-15T03:00:30.764Z 19 Acked 1018 Write 2 Done Sent Done false
74262024-02-15T03:00:30.765Z 20 Acked 1019 Write 8 Done Sent Done false
74272024-02-15T03:00:30.765Z 21 Acked 1020 Flush 0 Done Sent Done false
74282024-02-15T03:00:30.766Z 22 Acked 1021 Write 5 Done Sent Done false
74292024-02-15T03:00:30.766Z 23 Acked 1022 Read 2 Done Sent Done false
74302024-02-15T03:00:30.766Z 24 Acked 1023 Write 9 Done Sent Done false
74312024-02-15T03:00:30.767Z 25 Acked 1024 Write 7 Done Sent Done false
74322024-02-15T03:00:30.767Z 26 Acked 1025 Write 9 Done Sent Done false
74332024-02-15T03:00:30.767Z 27 Acked 1026 Write 6 Done Sent Done false
74342024-02-15T03:00:30.767Z 28 Acked 1027 Read 10 Done Sent Done false
74352024-02-15T03:00:30.767Z 29 Acked 1028 Read 4 Done Sent Done false
74362024-02-15T03:00:30.767Z 30 Acked 1029 Read 8 Done Sent Done false
74372024-02-15T03:00:30.768Z STATES DS:0 DS:1 DS:2 TOTAL
74382024-02-15T03:00:30.768Z New 0 0 0 0
74392024-02-15T03:00:30.768Z Sent 0 28 0 28
74402024-02-15T03:00:30.768Z Done 30 2 30 62
74412024-02-15T03:00:30.768Z Skipped 0 0 0 0
74422024-02-15T03:00:30.769Z Error 0 0 0 0
74432024-02-15T03:00:30.769ZLast Flush: 1020 0 1020
74442024-02-15T03:00:30.769ZDownstairs last five completed:
74452024-02-15T03:00:30.769ZUpstairs last five completed: 30 29 28 27 26
74462024-02-15T03:00:30.769Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:29.662881347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1399}
74472024-02-15T03:00:30.770ZWrote out file "/tmp/test_repair_verify.data"
74482024-02-15T03:00:30.770Z
74492024-02-15T03:00:30.770ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74502024-02-15T03:00:30.796ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74512024-02-15T03:00:30.797Z 0 000-019 62 62 62 109 109 109 F F F
74522024-02-15T03:00:30.797Z 1 020-039 57 57 57 103 103 103 F F F
74532024-02-15T03:00:30.797Z 2 040-059 65 65 65 120 120 120 F F F
74542024-02-15T03:00:30.797Z 3 060-079 66 59 66 121 107 121 F F F <---
74552024-02-15T03:00:30.797Z 4 080-099 66 54 66 121 99 121 F F F <---
74562024-02-15T03:00:30.798Z 5 100-119 51 51 51 94 94 94 F F F
74572024-02-15T03:00:30.798Z 6 120-139 64 64 64 119 119 119 F F F
74582024-02-15T03:00:30.798Z 7 140-159 66 64 66 121 119 121 F F F <---
74592024-02-15T03:00:30.798Z 8 160-179 66 64 66 121 119 121 F F F <---
74602024-02-15T03:00:30.798Z 9 180-199 63 63 63 115 115 115 F F F
74612024-02-15T03:00:30.799Z 10 200-219 64 64 64 118 118 118 F F F
74622024-02-15T03:00:30.799Z 11 220-239 66 57 66 121 103 121 T F T <---
74632024-02-15T03:00:30.799Z 12 240-259 66 65 66 121 120 121 F F F <---
74642024-02-15T03:00:30.799Z 13 260-279 66 64 66 121 119 121 F F F <---
74652024-02-15T03:00:30.799Z 14 280-299 64 64 64 116 116 116 F F F
74662024-02-15T03:00:30.800Z 15 300-319 66 64 66 121 119 121 F T F <---
74672024-02-15T03:00:30.800Z 16 320-339 64 64 64 118 118 118 F F F
74682024-02-15T03:00:30.800Z 17 340-359 64 64 64 116 116 116 F F F
74692024-02-15T03:00:30.800Z 18 360-379 65 65 65 120 120 120 F F F
74702024-02-15T03:00:30.800Z 19 380-399 64 64 64 119 119 119 T F T <---
74712024-02-15T03:00:30.801Z 20 400-419 62 62 62 110 110 110 T F T <---
74722024-02-15T03:00:30.801Z 21 420-439 62 62 62 110 110 110 T F T <---
74732024-02-15T03:00:30.801Z 22 440-459 60 60 60 108 108 108 F F F
74742024-02-15T03:00:30.801Z 23 460-479 66 63 66 121 115 121 F F F <---
74752024-02-15T03:00:30.801Z 24 480-499 66 60 66 121 108 121 F F F <---
74762024-02-15T03:00:30.802Z 25 500-519 59 59 59 107 107 107 F F F
74772024-02-15T03:00:30.802Z 26 520-539 66 55 66 121 101 121 F F F <---
74782024-02-15T03:00:30.802Z 27 540-559 66 55 66 121 101 121 F F F <---
74792024-02-15T03:00:30.802Z 28 560-579 62 62 62 112 112 112 T F T <---
74802024-02-15T03:00:30.802Z 29 580-599 65 65 65 120 120 120 F F F
74812024-02-15T03:00:30.802ZMax gen: 66, Max flush: 121
74822024-02-15T03:00:30.803ZError: Difference in extent metadata found!
74832024-02-15T03:00:30.803ZA Difference in extent metadata is expected here
74842024-02-15T03:00:30.803ZOn loop 33
74852024-02-15T03:00:30.803Z
74862024-02-15T03:00:30.803ZVerifying data now
74872024-02-15T03:00:35.546ZNo change after verify
74882024-02-15T03:00:35.546ZLoop: 33 Downstairs dump after verify (and repair):
74892024-02-15T03:00:35.594ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74902024-02-15T03:00:35.594Z 0 000-019 62 62 62 109 109 109 F F F
74912024-02-15T03:00:35.595Z 1 020-039 57 57 57 103 103 103 F F F
74922024-02-15T03:00:35.595Z 2 040-059 65 65 65 120 120 120 F F F
74932024-02-15T03:00:35.595Z 3 060-079 66 66 66 121 121 121 F F F
74942024-02-15T03:00:35.595Z 4 080-099 66 66 66 121 121 121 F F F
74952024-02-15T03:00:35.595Z 5 100-119 51 51 51 94 94 94 F F F
74962024-02-15T03:00:35.595Z 6 120-139 64 64 64 119 119 119 F F F
74972024-02-15T03:00:35.596Z 7 140-159 66 66 66 121 121 121 F F F
74982024-02-15T03:00:35.596Z 8 160-179 66 66 66 121 121 121 F F F
74992024-02-15T03:00:35.596Z 9 180-199 63 63 63 115 115 115 F F F
75002024-02-15T03:00:35.596Z 10 200-219 64 64 64 118 118 118 F F F
75012024-02-15T03:00:35.596Z 11 220-239 67 67 67 122 122 122 F F F
75022024-02-15T03:00:35.597Z 12 240-259 66 66 66 121 121 121 F F F
75032024-02-15T03:00:35.597Z 13 260-279 66 66 66 121 121 121 F F F
75042024-02-15T03:00:35.597Z 14 280-299 64 64 64 116 116 116 F F F
75052024-02-15T03:00:35.597Z 15 300-319 66 66 66 121 121 121 F F F
75062024-02-15T03:00:35.597Z 16 320-339 64 64 64 118 118 118 F F F
75072024-02-15T03:00:35.597Z 17 340-359 64 64 64 116 116 116 F F F
75082024-02-15T03:00:35.598Z 18 360-379 65 65 65 120 120 120 F F F
75092024-02-15T03:00:35.598Z 19 380-399 67 67 67 122 122 122 F F F
75102024-02-15T03:00:35.598Z 20 400-419 67 67 67 122 122 122 F F F
75112024-02-15T03:00:35.623Z 21 420-439 67 67 67 122 122 122 F F F
75122024-02-15T03:00:35.623Z 22 440-459 60 60 60 108 108 108 F F F
75132024-02-15T03:00:35.623Z 23 460-479 66 66 66 121 121 121 F F F
75142024-02-15T03:00:35.623Z 24 480-499 66 66 66 121 121 121 F F F
75152024-02-15T03:00:35.624Z 25 500-519 59 59 59 107 107 107 F F F
75162024-02-15T03:00:35.624Z 26 520-539 66 66 66 121 121 121 F F F
75172024-02-15T03:00:35.624Z 27 540-559 66 66 66 121 121 121 F F F
75182024-02-15T03:00:35.624Z 28 560-579 67 67 67 122 122 122 F F F
75192024-02-15T03:00:35.624Z 29 580-599 65 65 65 120 120 120 F F F
75202024-02-15T03:00:35.624ZMax gen: 67, Max flush: 122
75212024-02-15T03:00:35.624Z
75222024-02-15T03:00:35.624ZBegin loop 34
75232024-02-15T03:00:35.625ZDownstairs to restart: 2
75242024-02-15T03:00:35.625Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.519776473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407}
75252024-02-15T03:00:35.625Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520169424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407}
75262024-02-15T03:00:35.625Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.52020512Z"Crucible runtime is spawned
75272024-02-15T03:00:35.625Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407}
75282024-02-15T03:00:35.625Z{"msg":"Crucible 2097c8e0-35b7-4b30-9f67-a9372ff8435b has session id: 4db602f1-c50e-463b-97c8-cef9858ce4d7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520316886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75292024-02-15T03:00:35.625Z{"msg":"Upstairs opts: Upstairs UUID: 2097c8e0-35b7-4b30-9f67-a9372ff8435b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520372389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75302024-02-15T03:00:35.625Z{"msg":"Crucible stats registered with UUID: 2097c8e0-35b7-4b30-9f67-a9372ff8435b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520401745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75312024-02-15T03:00:35.625Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520431851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407}
75322024-02-15T03:00:35.626Z{"msg":"2097c8e0-35b7-4b30-9f67-a9372ff8435b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.52051746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75332024-02-15T03:00:35.626Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520544877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75342024-02-15T03:00:35.626Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520560935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75352024-02-15T03:00:35.626Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520574653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75362024-02-15T03:00:35.626Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520624997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75372024-02-15T03:00:35.626Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520651234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75382024-02-15T03:00:35.626Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520674081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75392024-02-15T03:00:35.627Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520749581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75402024-02-15T03:00:35.627Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520830921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75412024-02-15T03:00:35.627Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520893383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75422024-02-15T03:00:35.627Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.520992631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75432024-02-15T03:00:35.628Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521028336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75442024-02-15T03:00:35.628Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:34.521061252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75452024-02-15T03:00:35.628Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:34.521108896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75462024-02-15T03:00:35.628Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:34.521142082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75472024-02-15T03:00:35.628Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521179628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75482024-02-15T03:00:35.628Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521218793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75492024-02-15T03:00:35.628Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521246519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75502024-02-15T03:00:35.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521619412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75512024-02-15T03:00:35.629Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521674675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75522024-02-15T03:00:35.629Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521704722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75532024-02-15T03:00:35.629Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.521730588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75542024-02-15T03:00:35.629Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.522250653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75552024-02-15T03:00:35.629Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.522392085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75562024-02-15T03:00:35.629Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.52275847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75572024-02-15T03:00:35.629Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.522814223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75582024-02-15T03:00:35.629Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.522848648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75592024-02-15T03:00:35.630Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.522883684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75602024-02-15T03:00:35.630Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.52291354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75612024-02-15T03:00:35.630Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:34.522930048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75622024-02-15T03:00:36.592Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.52158462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75632024-02-15T03:00:36.599Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.521719813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75642024-02-15T03:00:36.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.522573426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75652024-02-15T03:00:36.599Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.522615621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75662024-02-15T03:00:36.599Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523150214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75672024-02-15T03:00:36.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523703184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75682024-02-15T03:00:36.600Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523778075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75692024-02-15T03:00:36.600Z{"msg":"[0]R flush_numbers[0..12]: [109, 103, 120, 121, 121, 94, 119, 121, 121, 115, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.52381634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75702024-02-15T03:00:36.600Z{"msg":"[0]R generation[0..12]: [62, 57, 65, 66, 66, 51, 64, 66, 66, 63, 64, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523833978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75712024-02-15T03:00:36.600Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523850376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75722024-02-15T03:00:36.600Z{"msg":"[1]R flush_numbers[0..12]: [109, 103, 120, 121, 121, 94, 119, 121, 121, 115, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523873043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75732024-02-15T03:00:36.600Z{"msg":"[1]R generation[0..12]: [62, 57, 65, 66, 66, 51, 64, 66, 66, 63, 64, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523889191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75742024-02-15T03:00:36.600ZWait for a query_work_queue command to finish before sending IO
75752024-02-15T03:00:36.600Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523920687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75762024-02-15T03:00:36.600Z{"msg":"[2]R flush_numbers[0..12]: [109, 103, 120, 121, 121, 94, 119, 121, 121, 115, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523942944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75772024-02-15T03:00:36.600Z{"msg":"[2]R generation[0..12]: [62, 57, 65, 66, 66, 51, 64, 66, 66, 63, 64, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.523958512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75782024-02-15T03:00:36.600Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.52397997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75792024-02-15T03:00:36.600Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524008506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75802024-02-15T03:00:36.600Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524032593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75812024-02-15T03:00:36.600Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.52405821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75822024-02-15T03:00:36.600Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524079107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75832024-02-15T03:00:36.601Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524092675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75842024-02-15T03:00:36.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524106074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75852024-02-15T03:00:36.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524126981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75862024-02-15T03:00:36.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.52414076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75872024-02-15T03:00:36.601Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524161817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75882024-02-15T03:00:36.601Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524174545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75892024-02-15T03:00:36.601Z{"msg":"2097c8e0-35b7-4b30-9f67-a9372ff8435b is now active with session: 4db602f1-c50e-463b-97c8-cef9858ce4d7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524188674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75902024-02-15T03:00:36.601Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524207901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407,"session_id":"4db602f1-c50e-463b-97c8-cef9858ce4d7"}
75912024-02-15T03:00:36.601Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:35.524221119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407}
75922024-02-15T03:00:36.601ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75932024-02-15T03:00:36.601ZLoading write count information from file "/tmp/test_repair_verify.data"
75942024-02-15T03:00:36.601ZRun Repair workload
75952024-02-15T03:00:36.601Z01/30 Write block 81 len 8192 data: 6 5
75962024-02-15T03:00:36.601Z02/30 Read block 70 len 4096
75972024-02-15T03:00:36.605Z03/30 Read block 524 len 28672
75982024-02-15T03:00:36.605Z04/30 Read block 372 len 36864
75992024-02-15T03:00:36.605Z05/30 Read block 142 len 8192
76002024-02-15T03:00:36.605Z06/30 Write block 217 len 12288 data: 6 6 8
76012024-02-15T03:00:36.605Z07/30 Write block 384 len 24576 data: 11 8 6 7 6 7
76022024-02-15T03:00:36.605Z08/30 Flush
76032024-02-15T03:00:36.605Z09/30 Write block 590 len 36864 data: 6 8 6 7 9 7 5 5 5
76042024-02-15T03:00:36.606Z10/30 Read block 419 len 40960
76052024-02-15T03:00:36.609Z11/30 Write block 300 len 8192 data: 7 10
76062024-02-15T03:00:36.609Z12/30 Read block 319 len 28672
76072024-02-15T03:00:36.609Z13/30 Read block 587 len 28672
76082024-02-15T03:00:36.609Z14/30 Read block 397 len 8192
76092024-02-15T03:00:36.612Z15/30 Write block 79 len 32768 data: 5 4 7 6 4 4 4 4
76102024-02-15T03:00:36.613Z16/30 Write block 484 len 36864 data: 7 5 6 7 5 7 8 8 7
76112024-02-15T03:00:36.613Z17/30 Read block 244 len 32768
76122024-02-15T03:00:36.617Z18/30 Write block 554 len 28672 data: 6 5 5 7 7 8 7
76132024-02-15T03:00:36.617Z19/30 Read block 530 len 40960
76142024-02-15T03:00:36.617Z20/30 Read block 311 len 20480
76152024-02-15T03:00:36.617Z21/30 Read block 162 len 32768
76162024-02-15T03:00:36.617Z22/30 Read block 263 len 40960
76172024-02-15T03:00:36.625Z23/30 Write block 571 len 16384 data: 6 5 8 9
76182024-02-15T03:00:36.625Z24/30 Flush
76192024-02-15T03:00:36.625Z25/30 Write block 352 len 36864 data: 7 7 7 6 7 7 7 7 6
76202024-02-15T03:00:36.625Z26/30 Write block 144 len 4096 data: 8
76212024-02-15T03:00:36.625Z27/30 Read block 255 len 24576
76222024-02-15T03:00:36.625Z28/30 Read block 497 len 12288
76232024-02-15T03:00:36.625Z29/30 Read block 340 len 16384
76242024-02-15T03:00:36.626Z30/30 Read block 453 len 28672
76252024-02-15T03:00:36.636Z----------------------------------------------------------------
76262024-02-15T03:00:36.637Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
76272024-02-15T03:00:36.637ZStates: Active Active Active
76282024-02-15T03:00:36.637ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76292024-02-15T03:00:36.637Z 1 Acked 1000 Write 2 Done Done Done false
76302024-02-15T03:00:36.637Z 2 Acked 1001 Read 1 Done Done Done false
76312024-02-15T03:00:36.638Z 3 Acked 1002 Read 7 Done Done Done false
76322024-02-15T03:00:36.638Z 4 Acked 1003 Read 9 Done Done Done false
76332024-02-15T03:00:36.638Z 5 Acked 1004 Read 2 Done Done Sent false
76342024-02-15T03:00:36.638Z 6 Acked 1005 Write 3 Done Done Sent false
76352024-02-15T03:00:36.638Z 7 Acked 1006 Write 6 Done Done Sent false
76362024-02-15T03:00:36.638Z 8 Acked 1007 Flush 0 Done Done Sent false
76372024-02-15T03:00:36.639Z 9 Acked 1008 Write 9 Done Done Sent false
76382024-02-15T03:00:36.639Z 10 Acked 1009 Read 10 Done Done Sent false
76392024-02-15T03:00:36.639Z 11 Acked 1010 Write 2 Done Done Sent false
76402024-02-15T03:00:36.639Z 12 Acked 1011 Read 7 Done Done Sent false
76412024-02-15T03:00:36.639Z 13 Acked 1012 Read 7 Done Done Sent false
76422024-02-15T03:00:36.640Z 14 Acked 1013 Read 2 Done Done Sent false
76432024-02-15T03:00:36.640Z 15 Acked 1014 Write 8 Done Done Sent false
76442024-02-15T03:00:36.640Z 16 Acked 1015 Write 9 Done Done Sent false
76452024-02-15T03:00:36.640Z 17 Acked 1016 Read 8 Done Done Sent false
76462024-02-15T03:00:36.640Z 18 Acked 1017 Write 7 Done Done Sent false
76472024-02-15T03:00:36.640Z 19 Acked 1018 Read 10 Done Done Sent false
76482024-02-15T03:00:36.640Z 20 Acked 1019 Read 5 Done Done Sent false
76492024-02-15T03:00:36.641Z 21 Acked 1020 Read 8 Done Done Sent false
76502024-02-15T03:00:36.641Z 22 Acked 1021 Read 10 Done Done Sent false
76512024-02-15T03:00:36.641Z 23 Acked 1022 Write 4 Done Done Sent false
76522024-02-15T03:00:36.641Z 24 Acked 1023 Flush 0 Done Done Sent false
76532024-02-15T03:00:36.641Z 25 Acked 1024 Write 9 Done Done Sent false
76542024-02-15T03:00:36.641Z 26 Acked 1025 Write 1 Done Done Sent false
76552024-02-15T03:00:36.641Z 27 Acked 1026 Read 6 Done Done Sent false
76562024-02-15T03:00:36.641Z 28 Acked 1027 Read 3 Done Done Sent false
76572024-02-15T03:00:36.641Z 29 Acked 1028 Read 4 Done Done Sent false
76582024-02-15T03:00:36.641Z 30 Acked 1029 Read 7 Done Sent Sent false
76592024-02-15T03:00:36.641Z STATES DS:0 DS:1 DS:2 TOTAL
76602024-02-15T03:00:36.642Z New 0 0 0 0
76612024-02-15T03:00:36.642Z Sent 0 1 26 27
76622024-02-15T03:00:36.642Z Done 30 29 4 63
76632024-02-15T03:00:36.642Z Skipped 0 0 0 0
76642024-02-15T03:00:36.642Z Error 0 0 0 0
76652024-02-15T03:00:36.642ZLast Flush: 1023 1023 0
76662024-02-15T03:00:36.642ZDownstairs last five completed:
76672024-02-15T03:00:36.643ZUpstairs last five completed: 30 29 28 27 26
76682024-02-15T03:00:36.643Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:35.556677223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1407}
76692024-02-15T03:00:36.643ZWrote out file "/tmp/test_repair_verify.data"
76702024-02-15T03:00:36.643Z
76712024-02-15T03:00:36.651ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76722024-02-15T03:00:36.656ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76732024-02-15T03:00:36.656Z 0 000-019 62 62 62 109 109 109 F F F
76742024-02-15T03:00:36.656Z 1 020-039 57 57 57 103 103 103 F F F
76752024-02-15T03:00:36.656Z 2 040-059 65 65 65 120 120 120 F F F
76762024-02-15T03:00:36.656Z 3 060-079 68 68 66 124 124 121 F F F <---
76772024-02-15T03:00:36.656Z 4 080-099 68 68 66 124 124 121 F F T <---
76782024-02-15T03:00:36.656Z 5 100-119 51 51 51 94 94 94 F F F
76792024-02-15T03:00:36.656Z 6 120-139 64 64 64 119 119 119 F F F
76802024-02-15T03:00:36.656Z 7 140-159 66 66 66 121 121 121 T T F <---
76812024-02-15T03:00:36.656Z 8 160-179 66 66 66 121 121 121 F F F
76822024-02-15T03:00:36.657Z 9 180-199 63 63 63 115 115 115 F F F
76832024-02-15T03:00:36.657Z 10 200-219 68 68 64 123 123 118 F F F <---
76842024-02-15T03:00:36.657Z 11 220-239 67 67 67 122 122 122 F F F
76852024-02-15T03:00:36.657Z 12 240-259 66 66 66 121 121 121 F F F
76862024-02-15T03:00:36.657Z 13 260-279 66 66 66 121 121 121 F F F
76872024-02-15T03:00:36.657Z 14 280-299 64 64 64 116 116 116 F F F
76882024-02-15T03:00:36.657Z 15 300-319 68 68 66 124 124 121 F F F <---
76892024-02-15T03:00:36.657Z 16 320-339 64 64 64 118 118 118 F F F
76902024-02-15T03:00:36.657Z 17 340-359 64 64 64 116 116 116 T T F <---
76912024-02-15T03:00:36.657Z 18 360-379 65 65 65 120 120 120 T T F <---
76922024-02-15T03:00:36.657Z 19 380-399 68 68 67 123 123 122 F F F <---
76932024-02-15T03:00:36.657Z 20 400-419 67 67 67 122 122 122 F F F
76942024-02-15T03:00:36.657Z 21 420-439 67 67 67 122 122 122 F F F
76952024-02-15T03:00:36.657Z 22 440-459 60 60 60 108 108 108 F F F
76962024-02-15T03:00:36.658Z 23 460-479 66 66 66 121 121 121 F F F
76972024-02-15T03:00:36.658Z 24 480-499 68 68 66 124 124 121 F F F <---
76982024-02-15T03:00:36.658Z 25 500-519 59 59 59 107 107 107 F F F
76992024-02-15T03:00:36.658Z 26 520-539 66 66 66 121 121 121 F F F
77002024-02-15T03:00:36.658Z 27 540-559 68 68 66 124 124 121 F F F <---
77012024-02-15T03:00:36.658Z 28 560-579 68 68 67 124 124 122 F F F <---
77022024-02-15T03:00:36.658Z 29 580-599 68 68 65 124 124 120 F F F <---
77032024-02-15T03:00:36.658ZMax gen: 68, Max flush: 124
77042024-02-15T03:00:36.658ZError: Difference in extent metadata found!
77052024-02-15T03:00:36.664ZA Difference in extent metadata is expected here
77062024-02-15T03:00:36.664ZOn loop 34
77072024-02-15T03:00:36.664Z
77082024-02-15T03:00:36.664ZVerifying data now
77092024-02-15T03:00:41.282ZNo change after verify
77102024-02-15T03:00:41.283ZLoop: 34 Downstairs dump after verify (and repair):
77112024-02-15T03:00:41.305ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77122024-02-15T03:00:41.305Z 0 000-019 62 62 62 109 109 109 F F F
77132024-02-15T03:00:41.305Z 1 020-039 57 57 57 103 103 103 F F F
77142024-02-15T03:00:41.306Z 2 040-059 65 65 65 120 120 120 F F F
77152024-02-15T03:00:41.306Z 3 060-079 68 68 68 124 124 124 F F F
77162024-02-15T03:00:41.306Z 4 080-099 68 68 68 124 124 124 F F F
77172024-02-15T03:00:41.306Z 5 100-119 51 51 51 94 94 94 F F F
77182024-02-15T03:00:41.306Z 6 120-139 64 64 64 119 119 119 F F F
77192024-02-15T03:00:41.306Z 7 140-159 69 69 69 125 125 125 F F F
77202024-02-15T03:00:41.306Z 8 160-179 66 66 66 121 121 121 F F F
77212024-02-15T03:00:41.306Z 9 180-199 63 63 63 115 115 115 F F F
77222024-02-15T03:00:41.307Z 10 200-219 68 68 68 123 123 123 F F F
77232024-02-15T03:00:41.307Z 11 220-239 67 67 67 122 122 122 F F F
77242024-02-15T03:00:41.307Z 12 240-259 66 66 66 121 121 121 F F F
77252024-02-15T03:00:41.307Z 13 260-279 66 66 66 121 121 121 F F F
77262024-02-15T03:00:41.307Z 14 280-299 64 64 64 116 116 116 F F F
77272024-02-15T03:00:41.314Z 15 300-319 68 68 68 124 124 124 F F F
77282024-02-15T03:00:41.314Z 16 320-339 64 64 64 118 118 118 F F F
77292024-02-15T03:00:41.314Z 17 340-359 69 69 69 125 125 125 F F F
77302024-02-15T03:00:41.314Z 18 360-379 69 69 69 125 125 125 F F F
77312024-02-15T03:00:41.315Z 19 380-399 68 68 68 123 123 123 F F F
77322024-02-15T03:00:41.315Z 20 400-419 67 67 67 122 122 122 F F F
77332024-02-15T03:00:41.315Z 21 420-439 67 67 67 122 122 122 F F F
77342024-02-15T03:00:41.315Z 22 440-459 60 60 60 108 108 108 F F F
77352024-02-15T03:00:41.315Z 23 460-479 66 66 66 121 121 121 F F F
77362024-02-15T03:00:41.315Z 24 480-499 68 68 68 124 124 124 F F F
77372024-02-15T03:00:41.316Z 25 500-519 59 59 59 107 107 107 F F F
77382024-02-15T03:00:41.316Z 26 520-539 66 66 66 121 121 121 F F F
77392024-02-15T03:00:41.316Z 27 540-559 68 68 68 124 124 124 F F F
77402024-02-15T03:00:41.316Z 28 560-579 68 68 68 124 124 124 F F F
77412024-02-15T03:00:41.316Z 29 580-599 68 68 68 124 124 124 F F F
77422024-02-15T03:00:41.316ZMax gen: 69, Max flush: 125
77432024-02-15T03:00:41.316Z
77442024-02-15T03:00:41.316ZBegin loop 35
77452024-02-15T03:00:41.317ZDownstairs to restart: 0
77462024-02-15T03:00:41.322Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250081253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415}
77472024-02-15T03:00:41.337Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250461955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415}
77482024-02-15T03:00:41.337ZCrucible runtime is spawned
77492024-02-15T03:00:41.337Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250517919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415}
77502024-02-15T03:00:41.337Z{"msg":"Crucible 2cf08834-c57d-48d4-8407-773a66747d5e has session id: c279b243-6db6-46c4-aaae-b20d8ec23b23","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250589139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77512024-02-15T03:00:41.337Z{"msg":"Upstairs opts: Upstairs UUID: 2cf08834-c57d-48d4-8407-773a66747d5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250630244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77522024-02-15T03:00:41.337Z{"msg":"Crucible stats registered with UUID: 2cf08834-c57d-48d4-8407-773a66747d5e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250655521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77532024-02-15T03:00:41.338Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250678128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415}
77542024-02-15T03:00:41.338Z{"msg":"2cf08834-c57d-48d4-8407-773a66747d5e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250709824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77552024-02-15T03:00:41.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.25074705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77562024-02-15T03:00:41.338Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250795624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77572024-02-15T03:00:41.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.25082227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77582024-02-15T03:00:41.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250868564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77592024-02-15T03:00:41.339Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250897671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77602024-02-15T03:00:41.339Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250931157Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77612024-02-15T03:00:41.339Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.250955194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77622024-02-15T03:00:41.339Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251008817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77632024-02-15T03:00:41.340Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251049972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77642024-02-15T03:00:41.340Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:40.251081088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77652024-02-15T03:00:41.340Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:40.251109424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77662024-02-15T03:00:41.340Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251134491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77672024-02-15T03:00:41.340Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:40.251162768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77682024-02-15T03:00:41.341Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251186605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77692024-02-15T03:00:41.341Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.25122596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77702024-02-15T03:00:41.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251263185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77712024-02-15T03:00:41.341Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251299501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77722024-02-15T03:00:41.341Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251750034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77732024-02-15T03:00:41.341Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.25178498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77742024-02-15T03:00:41.342Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251813156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77752024-02-15T03:00:41.342Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.251856231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77762024-02-15T03:00:41.342Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.252492551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77772024-02-15T03:00:41.342Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.252528216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77782024-02-15T03:00:41.342Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.253156988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77792024-02-15T03:00:41.342Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.253193253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77802024-02-15T03:00:41.343Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.253208821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77812024-02-15T03:00:41.343Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.25322181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77822024-02-15T03:00:41.343Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.253236148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77832024-02-15T03:00:41.343Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:40.253249716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77842024-02-15T03:00:42.379Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.252784287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77852024-02-15T03:00:42.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.252870537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77862024-02-15T03:00:42.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.253759036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77872024-02-15T03:00:42.424Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.253816678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77882024-02-15T03:00:42.425Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.254333033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77892024-02-15T03:00:42.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.25491863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77902024-02-15T03:00:42.425Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.254961475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77912024-02-15T03:00:42.425Z{"msg":"[0]R flush_numbers[0..12]: [109, 103, 120, 124, 124, 94, 119, 125, 121, 115, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.254991211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77922024-02-15T03:00:42.426Z{"msg":"[0]R generation[0..12]: [62, 57, 65, 68, 68, 51, 64, 69, 66, 63, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255014988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77932024-02-15T03:00:42.426Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255073001Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77942024-02-15T03:00:42.426Z{"msg":"[1]R flush_numbers[0..12]: [109, 103, 120, 124, 124, 94, 119, 125, 121, 115, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255116965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77952024-02-15T03:00:42.426Z{"msg":"[1]R generation[0..12]: [62, 57, 65, 68, 68, 51, 64, 69, 66, 63, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255133353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77962024-02-15T03:00:42.426Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.25515595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77972024-02-15T03:00:42.427Z{"msg":"[2]R flush_numbers[0..12]: [109, 103, 120, 124, 124, 94, 119, 125, 121, 115, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255171248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
77982024-02-15T03:00:42.427Z{"msg":"[2]R generation[0..12]: [62, 57, 65, 68, 68, 51, 64, 69, 66, 63, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255186486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
77992024-02-15T03:00:42.427Zc279b243-6db6-46c4-aaae-b20d8ec23b23"}
78002024-02-15T03:00:42.427Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255213823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78012024-02-15T03:00:42.427Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.25523592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78022024-02-15T03:00:42.428Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255251269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78032024-02-15T03:00:42.428Z{"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255271946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78042024-02-15T03:00:42.428Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255285514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78052024-02-15T03:00:42.428Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255300032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78062024-02-15T03:00:42.428Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255312311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78072024-02-15T03:00:42.428Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255326459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78082024-02-15T03:00:42.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255340067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78092024-02-15T03:00:42.429Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255356985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78102024-02-15T03:00:42.429Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255378893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78112024-02-15T03:00:42.429Z{"msg":"2cf08834-c57d-48d4-8407-773a66747d5e is now active with session: c279b243-6db6-46c4-aaae-b20d8ec23b23","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255409958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78122024-02-15T03:00:42.429Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255424857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415,"session_id":"c279b243-6db6-46c4-aaae-b20d8ec23b23"}
78132024-02-15T03:00:42.430Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:41.255456153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415}
78142024-02-15T03:00:42.430ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78152024-02-15T03:00:42.430ZLoading write count information from file "/tmp/test_repair_verify.data"
78162024-02-15T03:00:42.430ZRun Repair workload
78172024-02-15T03:00:42.430Z01/30 Write block 289 len 8192 data: 5 9
78182024-02-15T03:00:42.431Z02/30 Write block 132 len 40960 data: 6 7 7 7 8 7 7 6 8 9
78192024-02-15T03:00:42.431Z03/30 Read block 513 len 40960
78202024-02-15T03:00:42.431Z04/30 Read block 192 len 36864
78212024-02-15T03:00:42.431Z05/30 Read block 51 len 12288
78222024-02-15T03:00:42.431Z06/30 Write block 111 len 24576 data: 4 4 5 4 5 5
78232024-02-15T03:00:42.431Z07/30 Read block 427 len 4096
78242024-02-15T03:00:42.432Z08/30 Write block 450 len 32768 data: 3 3 2 2 2 2 2 2
78252024-02-15T03:00:42.432Z09/30 Read block 479 len 16384
78262024-02-15T03:00:42.432Z10/30 Read block 65 len 36864
78272024-02-15T03:00:42.432Z11/30 Write block 251 len 16384 data: 6 7 6 7
78282024-02-15T03:00:42.433Z12/30 Write block 10 len 20480 data: 11 7 5 4 4
78292024-02-15T03:00:42.433Z13/30 Read block 400 len 24576
78302024-02-15T03:00:42.433Z14/30 Read block 260 len 20480
78312024-02-15T03:00:42.433Z15/30 Read block 117 len 36864
78322024-02-15T03:00:42.433Z16/30 Write block 555 len 4096 data: 6
78332024-02-15T03:00:42.433Z17/30 Write block 165 len 32768 data: 6 7 7 6 5 6 6 5
78342024-02-15T03:00:42.434Z18/30 Write block 33 len 8192 data: 6 6
78352024-02-15T03:00:42.434Z19/30 Write block 413 len 36864 data: 7 7 6 6 4 4 5 5 5
78362024-02-15T03:00:42.434Z20/30 Read block 463 len 20480
78372024-02-15T03:00:42.434Z21/30 Read block 208 len 28672
78382024-02-15T03:00:42.434Z22/30 Write block 291 len 12288 data: 8 8 9
78392024-02-15T03:00:42.434Z23/30 Flush
78402024-02-15T03:00:42.434Z24/30 Write block 460 len 12288 data: 6 6 6
78412024-02-15T03:00:42.434Z25/30 Write block 24 len 20480 data: 3 2 3 4 5
78422024-02-15T03:00:42.434Z26/30 Write block 537 len 28672 data: 7 6 6 5 6 8 8
78432024-02-15T03:00:42.434Z27/30 Write block 371 len 4096 data: 6
78442024-02-15T03:00:42.434Z28/30 Read block 260 len 12288
78452024-02-15T03:00:42.435Z29/30 Read block 2 len 40960
78462024-02-15T03:00:42.435Z30/30 Read block 83 len 32768
78472024-02-15T03:00:42.435Z----------------------------------------------------------------
78482024-02-15T03:00:42.435Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
78492024-02-15T03:00:42.435ZStates: Active Active Active
78502024-02-15T03:00:42.435ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78512024-02-15T03:00:42.435Z 1 Acked 1000 Write 2 Done Done Done false
78522024-02-15T03:00:42.435Z 2 Acked 1001 Write 10 Done Done Done false
78532024-02-15T03:00:42.435Z 3 Acked 1002 Read 10 Done Done Done false
78542024-02-15T03:00:42.435Z 4 Acked 1003 Read 9 Done Done Done false
78552024-02-15T03:00:42.436Z 5 Acked 1004 Read 3 Sent Done Done false
78562024-02-15T03:00:42.436Z 6 Acked 1005 Write 6 Sent Done Done false
78572024-02-15T03:00:42.436Z 7 Acked 1006 Read 1 Sent Done Done false
78582024-02-15T03:00:42.436Z 8 Acked 1007 Write 8 Sent Done Done false
78592024-02-15T03:00:42.436Z 9 Acked 1008 Read 4 Sent Done Done false
78602024-02-15T03:00:42.436Z 10 Acked 1009 Read 9 Sent Done Done false
78612024-02-15T03:00:42.436Z 11 Acked 1010 Write 4 Sent Done Done false
78622024-02-15T03:00:42.436Z 12 Acked 1011 Write 5 Sent Done Done false
78632024-02-15T03:00:42.436Z 13 Acked 1012 Read 6 Sent Done Done false
78642024-02-15T03:00:42.436Z 14 Acked 1013 Read 5 Sent Done Done false
78652024-02-15T03:00:42.437Z 15 Acked 1014 Read 9 Sent Done Done false
78662024-02-15T03:00:42.437Z 16 Acked 1015 Write 1 Sent Done Done false
78672024-02-15T03:00:42.437Z 17 Acked 1016 Write 8 Sent Done Done false
78682024-02-15T03:00:42.437Z 18 Acked 1017 Write 2 Sent Done Done false
78692024-02-15T03:00:42.437Z 19 Acked 1018 Write 9 Sent Done Done false
78702024-02-15T03:00:42.437Z 20 Acked 1019 Read 5 Sent Done Done false
78712024-02-15T03:00:42.437Z 21 Acked 1020 Read 7 Sent Done Done false
78722024-02-15T03:00:42.437Z 22 Acked 1021 Write 3 Sent Done Done false
78732024-02-15T03:00:42.437Z 23 Acked 1022 Flush 0 Sent Done Done false
78742024-02-15T03:00:42.437Z 24 Acked 1023 Write 3 Sent Done Done false
78752024-02-15T03:00:42.437Z 25 Acked 1024 Write 5 Sent Done Done false
78762024-02-15T03:00:42.437Z 26 Acked 1025 Write 7 Sent Done Done false
78772024-02-15T03:00:42.437Z 27 Acked 1026 Write 1 Sent Done Done false
78782024-02-15T03:00:42.438Z 28 Acked 1027 Read 3 Sent Done Done false
78792024-02-15T03:00:42.438Z 29 Acked 1028 Read 10 Sent Done Done false
78802024-02-15T03:00:42.438Z 30 Acked 1029 Read 8 Sent Done Sent false
78812024-02-15T03:00:42.438Z STATES DS:0 DS:1 DS:2 TOTAL
78822024-02-15T03:00:42.438Z New 0 0 0 0
78832024-02-15T03:00:42.438Z Sent 26 0 1 27
78842024-02-15T03:00:42.438Z Done 4 30 29 63
78852024-02-15T03:00:42.438Z Skipped 0 0 0 0
78862024-02-15T03:00:42.460Z Error 0 0 0 0
78872024-02-15T03:00:42.460ZLast Flush: 0 1022 1022
78882024-02-15T03:00:42.461ZDownstairs last five completed:
78892024-02-15T03:00:42.461ZUpstairs last five completed: 30 29 28 27 26
78902024-02-15T03:00:42.461Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:41.287416878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1415}
78912024-02-15T03:00:42.461ZWrote out file "/tmp/test_repair_verify.data"
78922024-02-15T03:00:42.461Z
78932024-02-15T03:00:42.461ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78942024-02-15T03:00:42.461ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78952024-02-15T03:00:42.461Z 0 000-019 62 70 70 109 126 126 F F F <---
78962024-02-15T03:00:42.461Z 1 020-039 57 70 70 103 126 126 F T T <---
78972024-02-15T03:00:42.461Z 2 040-059 65 65 65 120 120 120 F F F
78982024-02-15T03:00:42.461Z 3 060-079 68 68 68 124 124 124 F F F
78992024-02-15T03:00:42.461Z 4 080-099 68 68 68 124 124 124 F F F
79002024-02-15T03:00:42.462Z 5 100-119 51 70 70 94 126 126 F F F <---
79012024-02-15T03:00:42.462Z 6 120-139 64 70 70 119 126 126 T F F <---
79022024-02-15T03:00:42.462Z 7 140-159 69 70 70 125 126 126 T F F <---
79032024-02-15T03:00:42.462Z 8 160-179 66 70 70 121 126 126 F F F <---
79042024-02-15T03:00:42.462Z 9 180-199 63 63 63 115 115 115 F F F
79052024-02-15T03:00:42.462Z 10 200-219 68 68 68 123 123 123 F F F
79062024-02-15T03:00:42.462Z 11 220-239 67 67 67 122 122 122 F F F
79072024-02-15T03:00:42.462Z 12 240-259 66 70 70 121 126 126 F F F <---
79082024-02-15T03:00:42.462Z 13 260-279 66 66 66 121 121 121 F F F
79092024-02-15T03:00:42.462Z 14 280-299 64 70 70 116 126 126 T F F <---
79102024-02-15T03:00:42.462Z 15 300-319 68 68 68 124 124 124 F F F
79112024-02-15T03:00:42.462Z 16 320-339 64 64 64 118 118 118 F F F
79122024-02-15T03:00:42.462Z 17 340-359 69 69 69 125 125 125 F F F
79132024-02-15T03:00:42.463Z 18 360-379 69 69 69 125 125 125 F T T <---
79142024-02-15T03:00:42.463Z 19 380-399 68 68 68 123 123 123 F F F
79152024-02-15T03:00:42.463Z 20 400-419 67 70 70 122 126 126 F F F <---
79162024-02-15T03:00:42.463Z 21 420-439 67 70 70 122 126 126 F F F <---
79172024-02-15T03:00:42.463Z 22 440-459 60 70 70 108 126 126 F F F <---
79182024-02-15T03:00:42.463Z 23 460-479 66 66 66 121 121 121 F T T <---
79192024-02-15T03:00:42.463Z 24 480-499 68 68 68 124 124 124 F F F
79202024-02-15T03:00:42.463Z 25 500-519 59 59 59 107 107 107 F F F
79212024-02-15T03:00:42.463Z 26 520-539 66 66 66 121 121 121 F T T <---
79222024-02-15T03:00:42.463Z 27 540-559 68 70 70 124 126 126 F T T <---
79232024-02-15T03:00:42.463Z 28 560-579 68 68 68 124 124 124 F F F
79242024-02-15T03:00:42.464Z 29 580-599 68 68 68 124 124 124 F F F
79252024-02-15T03:00:42.464ZMax gen: 70, Max flush: 126
79262024-02-15T03:00:42.464ZError: Difference in extent metadata found!
79272024-02-15T03:00:42.464ZA Difference in extent metadata is expected here
79282024-02-15T03:00:42.464ZOn loop 35
79292024-02-15T03:00:42.464Z
79302024-02-15T03:00:42.464ZVerifying data now
79312024-02-15T03:00:48.128ZNo change after verify
79322024-02-15T03:00:48.128ZLoop: 35 Downstairs dump after verify (and repair):
79332024-02-15T03:00:48.144ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79342024-02-15T03:00:48.144Z 0 000-019 70 70 70 126 126 126 F F F
79352024-02-15T03:00:48.144Z 1 020-039 71 71 71 127 127 127 F F F
79362024-02-15T03:00:48.144Z 2 040-059 65 65 65 120 120 120 F F F
79372024-02-15T03:00:48.144Z 3 060-079 68 68 68 124 124 124 F F F
79382024-02-15T03:00:48.145Z 4 080-099 68 68 68 124 124 124 F F F
79392024-02-15T03:00:48.145Z 5 100-119 70 70 70 126 126 126 F F F
79402024-02-15T03:00:48.145Z 6 120-139 70 70 70 126 126 126 F F F
79412024-02-15T03:00:48.145Z 7 140-159 70 70 70 126 126 126 F F F
79422024-02-15T03:00:48.145Z 8 160-179 70 70 70 126 126 126 F F F
79432024-02-15T03:00:48.145Z 9 180-199 63 63 63 115 115 115 F F F
79442024-02-15T03:00:48.145Z 10 200-219 68 68 68 123 123 123 F F F
79452024-02-15T03:00:48.151Z 11 220-239 67 67 67 122 122 122 F F F
79462024-02-15T03:00:48.151Z 12 240-259 70 70 70 126 126 126 F F F
79472024-02-15T03:00:48.151Z 13 260-279 66 66 66 121 121 121 F F F
79482024-02-15T03:00:48.151Z 14 280-299 70 70 70 126 126 126 F F F
79492024-02-15T03:00:48.151Z 15 300-319 68 68 68 124 124 124 F F F
79502024-02-15T03:00:48.151Z 16 320-339 64 64 64 118 118 118 F F F
79512024-02-15T03:00:48.151Z 17 340-359 69 69 69 125 125 125 F F F
79522024-02-15T03:00:48.151Z 18 360-379 71 71 71 127 127 127 F F F
79532024-02-15T03:00:48.152Z 19 380-399 68 68 68 123 123 123 F F F
79542024-02-15T03:00:48.152Z 20 400-419 70 70 70 126 126 126 F F F
79552024-02-15T03:00:48.152Z 21 420-439 70 70 70 126 126 126 F F F
79562024-02-15T03:00:48.152Z 22 440-459 70 70 70 126 126 126 F F F
79572024-02-15T03:00:48.152Z 23 460-479 71 71 71 127 127 127 F F F
79582024-02-15T03:00:48.152Z 24 480-499 68 68 68 124 124 124 F F F
79592024-02-15T03:00:48.152Z 25 500-519 59 59 59 107 107 107 F F F
79602024-02-15T03:00:48.152Z 26 520-539 71 71 71 127 127 127 F F F
79612024-02-15T03:00:48.152Z 27 540-559 71 71 71 127 127 127 F F F
79622024-02-15T03:00:48.153Z 28 560-579 68 68 68 124 124 124 F F F
79632024-02-15T03:00:48.153Z 29 580-599 68 68 68 124 124 124 F F F
79642024-02-15T03:00:48.153ZMax gen: 71, Max flush: 127
79652024-02-15T03:00:48.153Z
79662024-02-15T03:00:48.153ZBegin loop 36
79672024-02-15T03:00:48.153ZDownstairs to restart: 2
79682024-02-15T03:00:48.180Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092228193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423}
79692024-02-15T03:00:48.188Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092534984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423}
79702024-02-15T03:00:48.188ZCrucible runtime is spawned
79712024-02-15T03:00:48.188Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092579059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423}
79722024-02-15T03:00:48.188Z{"msg":"Crucible 5617f704-d43e-4b2e-be1a-ba3ea8880b8a has session id: 9b4b3edc-336c-46f4-a511-0e797a0562f9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092622673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79732024-02-15T03:00:48.188Z{"msg":"Upstairs opts: Upstairs UUID: 5617f704-d43e-4b2e-be1a-ba3ea8880b8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092681956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79742024-02-15T03:00:48.189Z{"msg":"Crucible stats registered with UUID: 5617f704-d43e-4b2e-be1a-ba3ea8880b8a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092714952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79752024-02-15T03:00:48.189Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092762636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423}
79762024-02-15T03:00:48.189Z{"msg":"5617f704-d43e-4b2e-be1a-ba3ea8880b8a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.09280736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79772024-02-15T03:00:48.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092843545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79782024-02-15T03:00:48.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092877991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79792024-02-15T03:00:48.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092922675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79802024-02-15T03:00:48.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.092953332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79812024-02-15T03:00:48.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093025073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79822024-02-15T03:00:48.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093068387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79832024-02-15T03:00:48.189Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093115891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79842024-02-15T03:00:48.189Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093173754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79852024-02-15T03:00:48.189Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093234496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79862024-02-15T03:00:48.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093268582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79872024-02-15T03:00:48.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093311897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79882024-02-15T03:00:48.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.093356051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79892024-02-15T03:00:48.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.094014069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79902024-02-15T03:00:48.189Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.094062753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79912024-02-15T03:00:48.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.094107877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79922024-02-15T03:00:48.189Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.094150442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79932024-02-15T03:00:48.189Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.094654918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79942024-02-15T03:00:48.189Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.094784092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79952024-02-15T03:00:48.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.09503875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79962024-02-15T03:00:48.189Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.095097413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79972024-02-15T03:00:48.189Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.095148067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79982024-02-15T03:00:48.190Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.095183892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
79992024-02-15T03:00:48.190Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.09519989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80002024-02-15T03:00:48.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.095355931Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80012024-02-15T03:00:48.190Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.095388637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80022024-02-15T03:00:48.190Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.095409154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80032024-02-15T03:00:48.190Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.095719865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80042024-02-15T03:00:48.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096200965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80052024-02-15T03:00:48.190Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.09623694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80062024-02-15T03:00:48.190Z{"msg":"[0]R flush_numbers[0..12]: [126, 127, 120, 124, 124, 126, 126, 126, 126, 115, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096253638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80072024-02-15T03:00:48.190Z{"msg":"[0]R generation[0..12]: [70, 71, 65, 68, 68, 70, 70, 70, 70, 63, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096268096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80082024-02-15T03:00:48.190Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096289854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80092024-02-15T03:00:48.190Z{"msg":"[1]R flush_numbers[0..12]: [126, 127, 120, 124, 124, 126, 126, 126, 126, 115, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096304802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80102024-02-15T03:00:48.190Z{"msg":"[1]R generation[0..12]: [70, 71, 65, 68, 68, 70, 70, 70, 70, 63, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096326229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80112024-02-15T03:00:48.190Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096341107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80122024-02-15T03:00:48.190Z{"msg":"[2]R flush_numbers[0..12]: [126, 127, 120, 124, 124, 126, 126, 126, 126, 115, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096367374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80132024-02-15T03:00:48.190Z{"msg":"[2]R generation[0..12]: [70, 71, 65, 68, 68, 70, 70, 70, 70, 63, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096381702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80142024-02-15T03:00:48.190Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.09639667ZWait for a query_work_queue command to finish before sending IO
80152024-02-15T03:00:48.190Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80162024-02-15T03:00:48.190Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096423987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80172024-02-15T03:00:48.190Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096445934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80182024-02-15T03:00:48.190Z{"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096459392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80192024-02-15T03:00:48.190Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096473131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80202024-02-15T03:00:48.190Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096486909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80212024-02-15T03:00:48.190Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096499207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80222024-02-15T03:00:48.190Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096536513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80232024-02-15T03:00:48.191Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.09655711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80242024-02-15T03:00:48.191Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096579547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80252024-02-15T03:00:48.191Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096593016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80262024-02-15T03:00:48.191Z{"msg":"5617f704-d43e-4b2e-be1a-ba3ea8880b8a is now active with session: 9b4b3edc-336c-46f4-a511-0e797a0562f9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096613213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80272024-02-15T03:00:48.191Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096625732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423,"session_id":"9b4b3edc-336c-46f4-a511-0e797a0562f9"}
80282024-02-15T03:00:48.191Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:47.096644669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423}
80292024-02-15T03:00:48.191ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80302024-02-15T03:00:48.191ZLoading write count information from file "/tmp/test_repair_verify.data"
80312024-02-15T03:00:48.191ZRun Repair workload
80322024-02-15T03:00:48.191Z01/30 Write block 283 len 32768 data: 4 6 8 9 8 6 6 10
80332024-02-15T03:00:48.191Z02/30 Read block 319 len 32768
80342024-02-15T03:00:48.191Z03/30 Read block 255 len 36864
80352024-02-15T03:00:48.191Z04/30 Write block 409 len 24576 data: 9 9 8 8 8 8
80362024-02-15T03:00:48.191Z05/30 Read block 171 len 4096
80372024-02-15T03:00:48.191Z06/30 Write block 195 len 40960 data: 9 8 7 5 4 5 5 4 4 4
80382024-02-15T03:00:48.191Z07/30 Write block 477 len 4096 data: 6
80392024-02-15T03:00:48.191Z08/30 Read block 254 len 16384
80402024-02-15T03:00:48.191Z09/30 Write block 241 len 8192 data: 7 7
80412024-02-15T03:00:48.191Z10/30 Read block 540 len 28672
80422024-02-15T03:00:48.191Z11/30 Read block 282 len 32768
80432024-02-15T03:00:48.191Z12/30 Read block 224 len 16384
80442024-02-15T03:00:48.191Z13/30 Write block 577 len 4096 data: 11
80452024-02-15T03:00:48.191Z14/30 Read block 521 len 12288
80462024-02-15T03:00:48.191Z15/30 Read block 473 len 24576
80472024-02-15T03:00:48.191Z16/30 Flush
80482024-02-15T03:00:48.196Z17/30 Write block 266 len 20480 data: 7 5 6 6 7
80492024-02-15T03:00:48.197Z18/30 Write block 331 len 4096 data: 10
80502024-02-15T03:00:48.197Z19/30 Read block 346 len 20480
80512024-02-15T03:00:48.197Z20/30 Write block 446 len 28672 data: 9 8 7 5 4 4 3
80522024-02-15T03:00:48.197Z21/30 Read block 188 len 20480
80532024-02-15T03:00:48.197Z22/30 Write block 449 len 8192 data: 6 5
80542024-02-15T03:00:48.197Z23/30 Read block 127 len 36864
80552024-02-15T03:00:48.197Z24/30 Read block 549 len 20480
80562024-02-15T03:00:48.197Z25/30 Write block 407 len 12288 data: 8 8 10
80572024-02-15T03:00:48.197Z26/30 Read block 576 len 36864
80582024-02-15T03:00:48.197Z27/30 Write block 115 len 16384 data: 6 6 7 6
80592024-02-15T03:00:48.197Z28/30 Write block 336 len 16384 data: 10 9 9 6
80602024-02-15T03:00:48.197Z29/30 Write block 334 len 8192 data: 12 12
80612024-02-15T03:00:48.197Z30/30 Write block 455 len 24576 data: 3 3 3 2 3 7
80622024-02-15T03:00:48.197Z----------------------------------------------------------------
80632024-02-15T03:00:48.197Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
80642024-02-15T03:00:48.197ZStates: Active Active Active
80652024-02-15T03:00:48.197ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80662024-02-15T03:00:48.197Z 1 Acked 1000 Write 8 Done Done Done false
80672024-02-15T03:00:48.197Z 2 Acked 1001 Read 8 Done Done Done false
80682024-02-15T03:00:48.197Z 3 Acked 1002 Read 9 Done Done Sent false
80692024-02-15T03:00:48.197Z 4 Acked 1003 Write 6 Done Done Sent false
80702024-02-15T03:00:48.197Z 5 Acked 1004 Read 1 Done Done Sent false
80712024-02-15T03:00:48.197Z 6 Acked 1005 Write 10 Done Done Sent false
80722024-02-15T03:00:48.198Z 7 Acked 1006 Write 1 Done Done Sent false
80732024-02-15T03:00:48.198Z 8 Acked 1007 Read 4 Done Done Sent false
80742024-02-15T03:00:48.198Z 9 Acked 1008 Write 2 Done Done Sent false
80752024-02-15T03:00:48.198Z 10 Acked 1009 Read 7 Done Done Sent false
80762024-02-15T03:00:48.198Z 11 Acked 1010 Read 8 Done Done Sent false
80772024-02-15T03:00:48.198Z 12 Acked 1011 Read 4 Done Done Sent false
80782024-02-15T03:00:48.198Z 13 Acked 1012 Write 1 Done Done Sent false
80792024-02-15T03:00:48.198Z 14 Acked 1013 Read 3 Done Done Sent false
80802024-02-15T03:00:48.198Z 15 Acked 1014 Read 6 Done Done Sent false
80812024-02-15T03:00:48.198Z 16 Acked 1015 Flush 0 Done Done Sent false
80822024-02-15T03:00:48.198Z 17 Acked 1016 Write 5 Done Done Sent false
80832024-02-15T03:00:48.198Z 18 Acked 1017 Write 1 Done Done Sent false
80842024-02-15T03:00:48.198Z 19 Acked 1018 Read 5 Done Done Sent false
80852024-02-15T03:00:48.198Z 20 Acked 1019 Write 7 Done Done Sent false
80862024-02-15T03:00:48.198Z 21 Acked 1020 Read 5 Done Done Sent false
80872024-02-15T03:00:48.198Z 22 Acked 1021 Write 2 Done Done Sent false
80882024-02-15T03:00:48.198Z 23 Acked 1022 Read 9 Done Done Sent false
80892024-02-15T03:00:48.198Z 24 Acked 1023 Read 5 Done Done Sent false
80902024-02-15T03:00:48.198Z 25 Acked 1024 Write 3 Done Done Sent false
80912024-02-15T03:00:48.198Z 26 Acked 1025 Read 9 Done Done Sent false
80922024-02-15T03:00:48.198Z 27 Acked 1026 Write 4 Done Done Sent false
80932024-02-15T03:00:48.198Z 28 Acked 1027 Write 4 Done Done Sent false
80942024-02-15T03:00:48.198Z 29 Acked 1028 Write 2 Done Done Sent false
80952024-02-15T03:00:48.198Z 30 Acked 1029 Write 6 Sent Sent Sent false
80962024-02-15T03:00:48.198Z STATES DS:0 DS:1 DS:2 TOTAL
80972024-02-15T03:00:48.199Z New 0 0 0 0
80982024-02-15T03:00:48.199Z Sent 1 1 28 30
80992024-02-15T03:00:48.199Z Done 29 29 2 60
81002024-02-15T03:00:48.199Z Skipped 0 0 0 0
81012024-02-15T03:00:48.199Z Error 0 0 0 0
81022024-02-15T03:00:48.199ZLast Flush: 1015 1015 0
81032024-02-15T03:00:48.199ZDownstairs last five completed:
81042024-02-15T03:00:48.199ZUpstairs last five completed: 30 29 28 27 26
81052024-02-15T03:00:48.199Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:47.124885601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1423}
81062024-02-15T03:00:48.202ZWrote out file "/tmp/test_repair_verify.data"
81072024-02-15T03:00:48.202Z
81082024-02-15T03:00:48.207ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81092024-02-15T03:00:48.222ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81102024-02-15T03:00:48.222Z 0 000-019 70 70 70 126 126 126 F F F
81112024-02-15T03:00:48.222Z 1 020-039 71 71 71 127 127 127 F F F
81122024-02-15T03:00:48.222Z 2 040-059 65 65 65 120 120 120 F F F
81132024-02-15T03:00:48.222Z 3 060-079 68 68 68 124 124 124 F F F
81142024-02-15T03:00:48.222Z 4 080-099 68 68 68 124 124 124 F F F
81152024-02-15T03:00:48.222Z 5 100-119 70 70 70 126 126 126 T T F <---
81162024-02-15T03:00:48.222Z 6 120-139 70 70 70 126 126 126 F F F
81172024-02-15T03:00:48.222Z 7 140-159 70 70 70 126 126 126 F F F
81182024-02-15T03:00:48.222Z 8 160-179 70 70 70 126 126 126 F F F
81192024-02-15T03:00:48.222Z 9 180-199 72 72 63 128 128 115 F F F <---
81202024-02-15T03:00:48.222Z 10 200-219 72 72 68 128 128 123 F F F <---
81212024-02-15T03:00:48.222Z 11 220-239 67 67 67 122 122 122 F F F
81222024-02-15T03:00:48.222Z 12 240-259 72 72 70 128 128 126 F F F <---
81232024-02-15T03:00:48.222Z 13 260-279 66 66 66 121 121 121 T T F <---
81242024-02-15T03:00:48.222Z 14 280-299 72 72 70 128 128 126 F F T <---
81252024-02-15T03:00:48.222Z 15 300-319 68 68 68 124 124 124 F F F
81262024-02-15T03:00:48.222Z 16 320-339 64 64 64 118 118 118 T T F <---
81272024-02-15T03:00:48.223Z 17 340-359 69 69 69 125 125 125 F F F
81282024-02-15T03:00:48.223Z 18 360-379 71 71 71 127 127 127 F F F
81292024-02-15T03:00:48.228Z 19 380-399 68 68 68 123 123 123 F F F
81302024-02-15T03:00:48.228Z 20 400-419 72 72 70 128 128 126 T T F <---
81312024-02-15T03:00:48.228Z 21 420-439 70 70 70 126 126 126 F F F
81322024-02-15T03:00:48.228Z 22 440-459 70 70 70 126 126 126 T T F <---
81332024-02-15T03:00:48.228Z 23 460-479 72 72 71 128 128 127 T T F <---
81342024-02-15T03:00:48.228Z 24 480-499 68 68 68 124 124 124 F F F
81352024-02-15T03:00:48.228Z 25 500-519 59 59 59 107 107 107 F F F
81362024-02-15T03:00:48.228Z 26 520-539 71 71 71 127 127 127 F F F
81372024-02-15T03:00:48.228Z 27 540-559 71 71 71 127 127 127 F F F
81382024-02-15T03:00:48.228Z 28 560-579 72 72 68 128 128 124 F F F <---
81392024-02-15T03:00:48.228Z 29 580-599 68 68 68 124 124 124 F F F
81402024-02-15T03:00:48.228ZMax gen: 72, Max flush: 128
81412024-02-15T03:00:48.228ZError: Difference in extent metadata found!
81422024-02-15T03:00:48.228ZA Difference in extent metadata is expected here
81432024-02-15T03:00:48.228ZOn loop 36
81442024-02-15T03:00:48.228Z
81452024-02-15T03:00:48.232ZVerifying data now
81462024-02-15T03:00:53.808ZNo change after verify
81472024-02-15T03:00:53.808ZLoop: 36 Downstairs dump after verify (and repair):
81482024-02-15T03:00:53.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81492024-02-15T03:00:53.824Z 0 000-019 70 70 70 126 126 126 F F F
81502024-02-15T03:00:53.825Z 1 020-039 71 71 71 127 127 127 F F F
81512024-02-15T03:00:53.825Z 2 040-059 65 65 65 120 120 120 F F F
81522024-02-15T03:00:53.825Z 3 060-079 68 68 68 124 124 124 F F F
81532024-02-15T03:00:53.841Z 4 080-099 68 68 68 124 124 124 F F F
81542024-02-15T03:00:53.841Z 5 100-119 73 73 73 129 129 129 F F F
81552024-02-15T03:00:53.841Z 6 120-139 70 70 70 126 126 126 F F F
81562024-02-15T03:00:53.841Z 7 140-159 70 70 70 126 126 126 F F F
81572024-02-15T03:00:53.841Z 8 160-179 70 70 70 126 126 126 F F F
81582024-02-15T03:00:53.841Z 9 180-199 72 72 72 128 128 128 F F F
81592024-02-15T03:00:53.841Z 10 200-219 72 72 72 128 128 128 F F F
81602024-02-15T03:00:53.841Z 11 220-239 67 67 67 122 122 122 F F F
81612024-02-15T03:00:53.841Z 12 240-259 72 72 72 128 128 128 F F F
81622024-02-15T03:00:53.842Z 13 260-279 73 73 73 129 129 129 F F F
81632024-02-15T03:00:53.842Z 14 280-299 72 72 72 128 128 128 F F F
81642024-02-15T03:00:53.842Z 15 300-319 68 68 68 124 124 124 F F F
81652024-02-15T03:00:53.842Z 16 320-339 73 73 73 129 129 129 F F F
81662024-02-15T03:00:53.842Z 17 340-359 69 69 69 125 125 125 F F F
81672024-02-15T03:00:53.842Z 18 360-379 71 71 71 127 127 127 F F F
81682024-02-15T03:00:53.842Z 19 380-399 68 68 68 123 123 123 F F F
81692024-02-15T03:00:53.842Z 20 400-419 73 73 73 129 129 129 F F F
81702024-02-15T03:00:53.842Z 21 420-439 70 70 70 126 126 126 F F F
81712024-02-15T03:00:53.843Z 22 440-459 73 73 73 129 129 129 F F F
81722024-02-15T03:00:53.843Z 23 460-479 73 73 73 129 129 129 F F F
81732024-02-15T03:00:53.843Z 24 480-499 68 68 68 124 124 124 F F F
81742024-02-15T03:00:53.843Z 25 500-519 59 59 59 107 107 107 F F F
81752024-02-15T03:00:53.843Z 26 520-539 71 71 71 127 127 127 F F F
81762024-02-15T03:00:53.843Z 27 540-559 71 71 71 127 127 127 F F F
81772024-02-15T03:00:53.843Z 28 560-579 72 72 72 128 128 128 F F F
81782024-02-15T03:00:53.843Z 29 580-599 68 68 68 124 124 124 F F F
81792024-02-15T03:00:53.843ZMax gen: 73, Max flush: 129
81802024-02-15T03:00:53.844Z
81812024-02-15T03:00:53.844ZBegin loop 37
81822024-02-15T03:00:53.844ZDownstairs to restart: 2
81832024-02-15T03:00:53.850Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.776754288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431}
81842024-02-15T03:00:53.850Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777122482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431}
81852024-02-15T03:00:53.850ZCrucible runtime is spawned
81862024-02-15T03:00:53.850Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777193373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431}
81872024-02-15T03:00:53.850Z{"msg":"Crucible 37a00b70-18e5-42a4-b7ea-32f911d6c1e1 has session id: af8ba326-c547-4a1c-972d-ed5b4ef2f15e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777235448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81882024-02-15T03:00:53.851Z{"msg":"Upstairs opts: Upstairs UUID: 37a00b70-18e5-42a4-b7ea-32f911d6c1e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777279473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81892024-02-15T03:00:53.851Z{"msg":"Crucible stats registered with UUID: 37a00b70-18e5-42a4-b7ea-32f911d6c1e1","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.77729906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81902024-02-15T03:00:53.851Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777329756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431}
81912024-02-15T03:00:53.851Z{"msg":"37a00b70-18e5-42a4-b7ea-32f911d6c1e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777358843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81922024-02-15T03:00:53.851Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.77738342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81932024-02-15T03:00:53.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777426524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81942024-02-15T03:00:53.851Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777484557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81952024-02-15T03:00:53.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.77753549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81962024-02-15T03:00:53.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777587474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81972024-02-15T03:00:53.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.77762383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81982024-02-15T03:00:53.873Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777665664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
81992024-02-15T03:00:53.873Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.77769741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82002024-02-15T03:00:53.873Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777742655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82012024-02-15T03:00:53.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777788179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82022024-02-15T03:00:53.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.777818495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82032024-02-15T03:00:53.874Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:52.77786035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82042024-02-15T03:00:53.874Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:52.777880127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82052024-02-15T03:00:53.875Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:52.777989704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82062024-02-15T03:00:53.875Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.778034128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82072024-02-15T03:00:53.875Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.778059905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82082024-02-15T03:00:53.875Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.778158143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82092024-02-15T03:00:53.875Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.778666099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82102024-02-15T03:00:53.875Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.778714533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82112024-02-15T03:00:53.875Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.778767166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82122024-02-15T03:00:53.875Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.778801262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82132024-02-15T03:00:53.875Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.779361532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82142024-02-15T03:00:53.876Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.779410476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82152024-02-15T03:00:53.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.779956797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82162024-02-15T03:00:53.876Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.779994193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82172024-02-15T03:00:53.876Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.780010531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82182024-02-15T03:00:53.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.780052435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82192024-02-15T03:00:53.876Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.780107218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82202024-02-15T03:00:53.877Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:52.780145563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82212024-02-15T03:00:54.862Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.778853818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82222024-02-15T03:00:54.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.778978743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82232024-02-15T03:00:54.888Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.779847784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82242024-02-15T03:00:54.888Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.779885689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82252024-02-15T03:00:54.888Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.780441539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82262024-02-15T03:00:54.888Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781057552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82272024-02-15T03:00:54.888Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781109956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82282024-02-15T03:00:54.888Z{"msg":"[0]R flush_numbers[0..12]: [126, 127, 120, 124, 124, 129, 126, 126, 126, 128, 128, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781140042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82292024-02-15T03:00:54.889Z{"msg":"[0]R generation[0..12]: [70, 71, 65, 68, 68, 73, 70, 70, 70, 72, 72, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781170198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82302024-02-15T03:00:54.889Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781189675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82312024-02-15T03:00:54.889Z{"msg":"[1]R flush_numbers[0..12]: [126, 127, 120, 124, 124, 129, 126, 126, 126, 128, 128, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781216332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82322024-02-15T03:00:54.889Z{"msg":"[1]R generation[0..12]: [70, 71, 65, 68, 68, 73, 70, 70, 70, 72, 72, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.78123488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82332024-02-15T03:00:54.889Z{"msg":"Wait for a query_work_queue command to finish before sending IO
82342024-02-15T03:00:54.889Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781265446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82352024-02-15T03:00:54.889Z{"msg":"[2]R flush_numbers[0..12]: [126, 127, 120, 124, 124, 129, 126, 126, 126, 128, 128, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781283824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82362024-02-15T03:00:54.889Z{"msg":"[2]R generation[0..12]: [70, 71, 65, 68, 68, 73, 70, 70, 70, 72, 72, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781301941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82372024-02-15T03:00:54.889Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781320959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82382024-02-15T03:00:54.889Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781353665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82392024-02-15T03:00:54.889Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781371723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82402024-02-15T03:00:54.889Z{"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.7813899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82412024-02-15T03:00:54.889Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781413658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82422024-02-15T03:00:54.889Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781432015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82432024-02-15T03:00:54.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781447413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82442024-02-15T03:00:54.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.78146956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82452024-02-15T03:00:54.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781494617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82462024-02-15T03:00:54.890Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781521644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82472024-02-15T03:00:54.890Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781540071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82482024-02-15T03:00:54.890Z{"msg":"37a00b70-18e5-42a4-b7ea-32f911d6c1e1 is now active with session: af8ba326-c547-4a1c-972d-ed5b4ef2f15e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781556549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82492024-02-15T03:00:54.891Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781572027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431,"session_id":"af8ba326-c547-4a1c-972d-ed5b4ef2f15e"}
82502024-02-15T03:00:54.891Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:53.781598594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431}
82512024-02-15T03:00:54.891ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82522024-02-15T03:00:54.891ZLoading write count information from file "/tmp/test_repair_verify.data"
82532024-02-15T03:00:54.891ZRun Repair workload
82542024-02-15T03:00:54.891Z01/30 Write block 466 len 16384 data: 7 7 8 9
82552024-02-15T03:00:54.891Z02/30 Read block 573 len 8192
82562024-02-15T03:00:54.891Z03/30 Read block 209 len 8192
82572024-02-15T03:00:54.891Z04/30 Read block 209 len 40960
82582024-02-15T03:00:54.891Z05/30 Read block 439 len 16384
82592024-02-15T03:00:54.891Z06/30 Write block 570 len 4096 data: 5
82602024-02-15T03:00:54.891Z07/30 Read block 556 len 8192
82612024-02-15T03:00:54.891Z08/30 Read block 528 len 20480
82622024-02-15T03:00:54.891Z09/30 Read block 374 len 4096
82632024-02-15T03:00:54.892Z10/30 Read block 443 len 8192
82642024-02-15T03:00:54.892Z11/30 Read block 439 len 12288
82652024-02-15T03:00:54.892Z12/30 Write block 547 len 36864 data: 4 5 4 4 4 4 6 7 7
82662024-02-15T03:00:54.892Z13/30 Read block 363 len 40960
82672024-02-15T03:00:54.892Z14/30 Read block 547 len 16384
82682024-02-15T03:00:54.892Z15/30 Write block 272 len 4096 data: 8
82692024-02-15T03:00:54.892Z16/30 Read block 442 len 28672
82702024-02-15T03:00:54.892Z17/30 Write block 96 len 28672 data: 8 10 10 8 7 8 8
82712024-02-15T03:00:54.892Z18/30 Read block 508 len 40960
82722024-02-15T03:00:54.892Z19/30 Write block 367 len 4096 data: 6
82732024-02-15T03:00:54.892Z20/30 Read block 424 len 8192
82742024-02-15T03:00:54.892Z21/30 Read block 230 len 12288
82752024-02-15T03:00:54.892Z22/30 Write block 177 len 40960 data: 6 6 6 6 5 4 6 7 8 8
82762024-02-15T03:00:54.892Z23/30 Write block 192 len 40960 data: 8 10 9 10 9 8 6 5 6 6
82772024-02-15T03:00:54.893Z24/30 Write block 350 len 16384 data: 6 7 8 8
82782024-02-15T03:00:54.893Z25/30 Read block 410 len 8192
82792024-02-15T03:00:54.893Z26/30 Read block 441 len 36864
82802024-02-15T03:00:54.893Z27/30 Read block 57 len 4096
82812024-02-15T03:00:54.893Z28/30 Read block 293 len 4096
82822024-02-15T03:00:54.893Z29/30 Read block 35 len 4096
82832024-02-15T03:00:54.893Z30/30 Read block 305 len 12288
82842024-02-15T03:00:54.893Z----------------------------------------------------------------
82852024-02-15T03:00:54.893Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
82862024-02-15T03:00:54.893ZStates: Active Active Active
82872024-02-15T03:00:54.893ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82882024-02-15T03:00:54.893Z 1 Acked 1000 Write 4 Done Done Done false
82892024-02-15T03:00:54.893Z 2 Acked 1001 Read 2 Done Done Done false
82902024-02-15T03:00:54.893Z 3 Acked 1002 Read 2 Done Done Done false
82912024-02-15T03:00:54.893Z 4 Acked 1003 Read 10 Done Done Done false
82922024-02-15T03:00:54.893Z 5 Acked 1004 Read 4 Done Done Sent false
82932024-02-15T03:00:54.893Z 6 Acked 1005 Write 1 Done Done Sent false
82942024-02-15T03:00:54.893Z 7 Acked 1006 Read 2 Done Done Sent false
82952024-02-15T03:00:54.893Z 8 Acked 1007 Read 5 Done Done Sent false
82962024-02-15T03:00:54.894Z 9 Acked 1008 Read 1 Done Done Sent false
82972024-02-15T03:00:54.894Z 10 Acked 1009 Read 2 Done Done Sent false
82982024-02-15T03:00:54.894Z 11 Acked 1010 Read 3 Done Done Sent false
82992024-02-15T03:00:54.894Z 12 Acked 1011 Write 9 Done Done Sent false
83002024-02-15T03:00:54.894Z 13 Acked 1012 Read 10 Done Done Sent false
83012024-02-15T03:00:54.894Z 14 Acked 1013 Read 4 Done Done Sent false
83022024-02-15T03:00:54.894Z 15 Acked 1014 Write 1 Done Done Sent false
83032024-02-15T03:00:54.894Z 16 Acked 1015 Read 7 Done Done Sent false
83042024-02-15T03:00:54.894Z 17 Acked 1016 Write 7 Done Done Sent false
83052024-02-15T03:00:54.894Z 18 Acked 1017 Read 10 Done Done Sent false
83062024-02-15T03:00:54.894Z 19 Acked 1018 Write 1 Done Done Sent false
83072024-02-15T03:00:54.894Z 20 Acked 1019 Read 2 Done Done Sent false
83082024-02-15T03:00:54.894Z 21 Acked 1020 Read 3 Done Done Sent false
83092024-02-15T03:00:54.894Z 22 Acked 1021 Write 10 Done Done Sent false
83102024-02-15T03:00:54.894Z 23 Acked 1022 Write 10 Done Done Sent false
83112024-02-15T03:00:54.894Z 24 Acked 1023 Write 4 Done Done Sent false
83122024-02-15T03:00:54.894Z 25 Acked 1024 Read 2 Done Done Sent false
83132024-02-15T03:00:54.894Z 26 Acked 1025 Read 9 Done Done Sent false
83142024-02-15T03:00:54.894Z 27 Acked 1026 Read 1 Done Done Sent false
83152024-02-15T03:00:54.894Z 28 Acked 1027 Read 1 Done Done Sent false
83162024-02-15T03:00:54.894Z 29 Acked 1028 Read 1 Done Done Sent false
83172024-02-15T03:00:54.894Z 30 Acked 1029 Read 3 Done Done Sent false
83182024-02-15T03:00:54.894Z STATES DS:0 DS:1 DS:2 TOTAL
83192024-02-15T03:00:54.894Z New 0 0 0 0
83202024-02-15T03:00:54.894Z Sent 0 0 26 26
83212024-02-15T03:00:54.895Z Done 30 30 4 64
83222024-02-15T03:00:54.905Z Skipped 0 0 0 0
83232024-02-15T03:00:54.905Z Error 0 0 0 0
83242024-02-15T03:00:54.905ZLast Flush: 0 0 0
83252024-02-15T03:00:54.905ZDownstairs last five completed:
83262024-02-15T03:00:54.905ZUpstairs last five completed: 30 29 28 27 26
83272024-02-15T03:00:54.905Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:53.807667098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1431}
83282024-02-15T03:00:54.905ZWrote out file "/tmp/test_repair_verify.data"
83292024-02-15T03:00:54.905Z
83302024-02-15T03:00:54.905ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83312024-02-15T03:00:54.913ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83322024-02-15T03:00:54.913Z 0 000-019 70 70 70 126 126 126 F F F
83332024-02-15T03:00:54.913Z 1 020-039 71 71 71 127 127 127 F F F
83342024-02-15T03:00:54.913Z 2 040-059 65 65 65 120 120 120 F F F
83352024-02-15T03:00:54.914Z 3 060-079 68 68 68 124 124 124 F F F
83362024-02-15T03:00:54.914Z 4 080-099 68 68 68 124 124 124 T T F <---
83372024-02-15T03:00:54.914Z 5 100-119 73 73 73 129 129 129 T T F <---
83382024-02-15T03:00:54.914Z 6 120-139 70 70 70 126 126 126 F F F
83392024-02-15T03:00:54.914Z 7 140-159 70 70 70 126 126 126 F F F
83402024-02-15T03:00:54.921Z 8 160-179 70 70 70 126 126 126 T T F <---
83412024-02-15T03:00:54.921Z 9 180-199 72 72 72 128 128 128 T T F <---
83422024-02-15T03:00:54.922Z 10 200-219 72 72 72 128 128 128 T T F <---
83432024-02-15T03:00:54.922Z 11 220-239 67 67 67 122 122 122 F F F
83442024-02-15T03:00:54.922Z 12 240-259 72 72 72 128 128 128 F F F
83452024-02-15T03:00:54.922Z 13 260-279 73 73 73 129 129 129 T T F <---
83462024-02-15T03:00:54.922Z 14 280-299 72 72 72 128 128 128 F F F
83472024-02-15T03:00:54.922Z 15 300-319 68 68 68 124 124 124 F F F
83482024-02-15T03:00:54.922Z 16 320-339 73 73 73 129 129 129 F F F
83492024-02-15T03:00:54.923Z 17 340-359 69 69 69 125 125 125 T T F <---
83502024-02-15T03:00:54.923Z 18 360-379 71 71 71 127 127 127 T T F <---
83512024-02-15T03:00:54.923Z 19 380-399 68 68 68 123 123 123 F F F
83522024-02-15T03:00:54.923Z 20 400-419 73 73 73 129 129 129 F F F
83532024-02-15T03:00:54.923Z 21 420-439 70 70 70 126 126 126 F F F
83542024-02-15T03:00:54.923Z 22 440-459 73 73 73 129 129 129 F F F
83552024-02-15T03:00:54.923Z 23 460-479 73 73 73 129 129 129 T T T
83562024-02-15T03:00:54.923Z 24 480-499 68 68 68 124 124 124 F F F
83572024-02-15T03:00:54.923Z 25 500-519 59 59 59 107 107 107 F F F
83582024-02-15T03:00:54.923Z 26 520-539 71 71 71 127 127 127 F F F
83592024-02-15T03:00:54.923Z 27 540-559 71 71 71 127 127 127 T T F <---
83602024-02-15T03:00:54.924Z 28 560-579 72 72 72 128 128 128 T T F <---
83612024-02-15T03:00:54.924Z 29 580-599 68 68 68 124 124 124 F F F
83622024-02-15T03:00:54.924ZMax gen: 73, Max flush: 129
83632024-02-15T03:00:54.924ZError: Difference in extent metadata found!
83642024-02-15T03:00:54.924ZA Difference in extent metadata is expected here
83652024-02-15T03:00:54.924ZOn loop 37
83662024-02-15T03:00:54.924Z
83672024-02-15T03:00:54.931ZVerifying data now
83682024-02-15T03:01:00.509ZNo change after verify
83692024-02-15T03:01:00.509ZLoop: 37 Downstairs dump after verify (and repair):
83702024-02-15T03:01:00.525ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83712024-02-15T03:01:00.525Z 0 000-019 70 70 70 126 126 126 F F F
83722024-02-15T03:01:00.525Z 1 020-039 71 71 71 127 127 127 F F F
83732024-02-15T03:01:00.525Z 2 040-059 65 65 65 120 120 120 F F F
83742024-02-15T03:01:00.526Z 3 060-079 68 68 68 124 124 124 F F F
83752024-02-15T03:01:00.526Z 4 080-099 74 74 74 130 130 130 F F F
83762024-02-15T03:01:00.526Z 5 100-119 74 74 74 130 130 130 F F F
83772024-02-15T03:01:00.526Z 6 120-139 70 70 70 126 126 126 F F F
83782024-02-15T03:01:00.526Z 7 140-159 70 70 70 126 126 126 F F F
83792024-02-15T03:01:00.537Z 8 160-179 74 74 74 130 130 130 F F F
83802024-02-15T03:01:00.537Z 9 180-199 74 74 74 130 130 130 F F F
83812024-02-15T03:01:00.537Z 10 200-219 74 74 74 130 130 130 F F F
83822024-02-15T03:01:00.537Z 11 220-239 67 67 67 122 122 122 F F F
83832024-02-15T03:01:00.537Z 12 240-259 72 72 72 128 128 128 F F F
83842024-02-15T03:01:00.537Z 13 260-279 74 74 74 130 130 130 F F F
83852024-02-15T03:01:00.537Z 14 280-299 72 72 72 128 128 128 F F F
83862024-02-15T03:01:00.537Z 15 300-319 68 68 68 124 124 124 F F F
83872024-02-15T03:01:00.537Z 16 320-339 73 73 73 129 129 129 F F F
83882024-02-15T03:01:00.537Z 17 340-359 74 74 74 130 130 130 F F F
83892024-02-15T03:01:00.537Z 18 360-379 74 74 74 130 130 130 F F F
83902024-02-15T03:01:00.538Z 19 380-399 68 68 68 123 123 123 F F F
83912024-02-15T03:01:00.538Z 20 400-419 73 73 73 129 129 129 F F F
83922024-02-15T03:01:00.538Z 21 420-439 70 70 70 126 126 126 F F F
83932024-02-15T03:01:00.538Z 22 440-459 73 73 73 129 129 129 F F F
83942024-02-15T03:01:00.538Z 23 460-479 74 74 74 130 130 130 F F F
83952024-02-15T03:01:00.538Z 24 480-499 68 68 68 124 124 124 F F F
83962024-02-15T03:01:00.538Z 25 500-519 59 59 59 107 107 107 F F F
83972024-02-15T03:01:00.538Z 26 520-539 71 71 71 127 127 127 F F F
83982024-02-15T03:01:00.538Z 27 540-559 74 74 74 130 130 130 F F F
83992024-02-15T03:01:00.538Z 28 560-579 74 74 74 130 130 130 F F F
84002024-02-15T03:01:00.538Z 29 580-599 68 68 68 124 124 124 F F F
84012024-02-15T03:01:00.538ZMax gen: 74, Max flush: 130
84022024-02-15T03:01:00.538Z
84032024-02-15T03:01:00.538ZBegin loop 38
84042024-02-15T03:01:00.538ZDownstairs to restart: 1
84052024-02-15T03:01:00.550Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.476643632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439}
84062024-02-15T03:01:00.550Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477051781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439}
84072024-02-15T03:01:00.550Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477082768Z"Crucible runtime is spawned
84082024-02-15T03:01:00.550Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439}
84092024-02-15T03:01:00.556Z{"msg":"Crucible 27a5fe70-858a-42b1-8f5f-347db3a0ac25 has session id: f119f5a1-3185-4be3-b742-7d6d86d3c682","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477131791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84102024-02-15T03:01:00.556Z{"msg":"Upstairs opts: Upstairs UUID: 27a5fe70-858a-42b1-8f5f-347db3a0ac25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477188884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84112024-02-15T03:01:00.556Z{"msg":"Crucible stats registered with UUID: 27a5fe70-858a-42b1-8f5f-347db3a0ac25","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477225819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84122024-02-15T03:01:00.556Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477243637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439}
84132024-02-15T03:01:00.557Z{"msg":"27a5fe70-858a-42b1-8f5f-347db3a0ac25 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477273164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84142024-02-15T03:01:00.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477292261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84152024-02-15T03:01:00.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477325677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84162024-02-15T03:01:00.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477369282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84172024-02-15T03:01:00.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477398598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84182024-02-15T03:01:00.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477423765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84192024-02-15T03:01:00.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477448342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84202024-02-15T03:01:00.557Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477472259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84212024-02-15T03:01:00.557Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477516263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84222024-02-15T03:01:00.557Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.47754394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84232024-02-15T03:01:00.557Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:59.477578475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84242024-02-15T03:01:00.557Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:59.477606252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84252024-02-15T03:01:00.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477632888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84262024-02-15T03:01:00.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477659815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84272024-02-15T03:01:00.558Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:00:59.47770006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84282024-02-15T03:01:00.558Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477723897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84292024-02-15T03:01:00.558Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477765842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84302024-02-15T03:01:00.558Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.477799988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84312024-02-15T03:01:00.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.4782595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84322024-02-15T03:01:00.558Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.478298795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84332024-02-15T03:01:00.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.478322492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84342024-02-15T03:01:00.558Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.478382155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84352024-02-15T03:01:00.558Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.478924947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84362024-02-15T03:01:00.558Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.478963382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84372024-02-15T03:01:00.559Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.479597642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84382024-02-15T03:01:00.559Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.479634518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84392024-02-15T03:01:00.559Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.479649876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84402024-02-15T03:01:00.559Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.479662394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84412024-02-15T03:01:00.559Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.479683432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84422024-02-15T03:01:00.559Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:00:59.47969759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84432024-02-15T03:01:01.564Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.479199026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84442024-02-15T03:01:01.564Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.479332809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84452024-02-15T03:01:01.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.480237185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84462024-02-15T03:01:01.600Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.480273051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84472024-02-15T03:01:01.600Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.480816943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84482024-02-15T03:01:01.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481466602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84492024-02-15T03:01:01.600Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481515875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84502024-02-15T03:01:01.600Z{"msg":"[0]R flush_numbers[0..12]: [126, 127, 120, 124, 130, 130, 126, 126, 130, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481545152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84512024-02-15T03:01:01.600Z{"msg":"[0]R generation[0..12]: [70, 71, 65, 68, 74, 74, 70, 70, 74, 74, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481588696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84522024-02-15T03:01:01.600Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481632791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84532024-02-15T03:01:01.601Z{"msg":"[1]R flush_numbers[0..12]: [126, 127, 120, 124, 130, 130, 126, 126, 130, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481675715Z"Wait for a query_work_queue command to finish before sending IO
84542024-02-15T03:01:01.601Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84552024-02-15T03:01:01.601Z{"msg":"[1]R generation[0..12]: [70, 71, 65, 68, 74, 74, 70, 70, 74, 74, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.48171967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84562024-02-15T03:01:01.601Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481761685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84572024-02-15T03:01:01.601Z{"msg":"[2]R flush_numbers[0..12]: [126, 127, 120, 124, 130, 130, 126, 126, 130, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.48179343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84582024-02-15T03:01:01.601Z{"msg":"[2]R generation[0..12]: [70, 71, 65, 68, 74, 74, 70, 70, 74, 74, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481825606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84592024-02-15T03:01:01.601Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.48187651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84602024-02-15T03:01:01.601Z{"msg":"Max found gen is 75","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481903267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84612024-02-15T03:01:01.601Z{"msg":"Generation requested: 76 >= found:75","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481928254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84622024-02-15T03:01:01.601Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.481967189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84632024-02-15T03:01:01.601Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482003184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84642024-02-15T03:01:01.602Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482046199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84652024-02-15T03:01:01.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482068546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84662024-02-15T03:01:01.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482083024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84672024-02-15T03:01:01.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.48211524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84682024-02-15T03:01:01.602Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482152556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84692024-02-15T03:01:01.602Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482189271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84702024-02-15T03:01:01.602Z{"msg":"27a5fe70-858a-42b1-8f5f-347db3a0ac25 is now active with session: f119f5a1-3185-4be3-b742-7d6d86d3c682","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482227146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84712024-02-15T03:01:01.602Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482262442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439,"session_id":"f119f5a1-3185-4be3-b742-7d6d86d3c682"}
84722024-02-15T03:01:01.602Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:00.482298307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439}
84732024-02-15T03:01:01.602ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84742024-02-15T03:01:01.602ZLoading write count information from file "/tmp/test_repair_verify.data"
84752024-02-15T03:01:01.602ZRun Repair workload
84762024-02-15T03:01:01.603Z01/30 Write block 426 len 28672 data: 7 7 6 5 5 5 4
84772024-02-15T03:01:01.603Z02/30 Write block 90 len 4096 data: 4
84782024-02-15T03:01:01.603Z03/30 Write block 409 len 20480 data: 11 10 9 9 9
84792024-02-15T03:01:01.603Z04/30 Write block 283 len 36864 data: 5 7 9 10 9 7 7 11 9
84802024-02-15T03:01:01.603Z05/30 Read block 443 len 28672
84812024-02-15T03:01:01.603Z06/30 Read block 239 len 24576
84822024-02-15T03:01:01.603Z07/30 Read block 579 len 20480
84832024-02-15T03:01:01.603Z08/30 Write block 258 len 8192 data: 6 7
84842024-02-15T03:01:01.603Z09/30 Flush
84852024-02-15T03:01:01.603Z10/30 Flush
84862024-02-15T03:01:01.603Z11/30 Write block 422 len 8192 data: 4 5
84872024-02-15T03:01:01.603Z12/30 Write block 162 len 12288 data: 9 6 4
84882024-02-15T03:01:01.604Z13/30 Flush
84892024-02-15T03:01:01.617Z14/30 Write block 341 len 32768 data: 4 4 4 5 6 7 6 6
84902024-02-15T03:01:01.617Z15/30 Read block 159 len 40960
84912024-02-15T03:01:01.617Z16/30 Write block 383 len 16384 data: 11 12 9 7
84922024-02-15T03:01:01.618Z17/30 Read block 141 len 20480
84932024-02-15T03:01:01.618Z18/30 Write block 569 len 24576 data: 3 6 7 6 9 10
84942024-02-15T03:01:01.618Z19/30 Write block 340 len 12288 data: 4 5 5
84952024-02-15T03:01:01.618Z20/30 Write block 104 len 4096 data: 6
84962024-02-15T03:01:01.618Z21/30 Flush
84972024-02-15T03:01:01.618Z22/30 Write block 399 len 16384 data: 5 4 3 3
84982024-02-15T03:01:01.618Z23/30 Read block 433 len 36864
84992024-02-15T03:01:01.618Z24/30 Read block 354 len 16384
85002024-02-15T03:01:01.619Z25/30 Write block 185 len 12288 data: 9 9 6
85012024-02-15T03:01:01.619Z26/30 Read block 23 len 32768
85022024-02-15T03:01:01.619Z27/30 Write block 47 len 36864 data: 10 9 8 5 3 5 6 6 7
85032024-02-15T03:01:01.619Z28/30 Write block 590 len 16384 data: 7 9 7 8
85042024-02-15T03:01:01.619Z29/30 Read block 356 len 12288
85052024-02-15T03:01:01.619Z30/30 Read block 326 len 40960
85062024-02-15T03:01:01.619Z----------------------------------------------------------------
85072024-02-15T03:01:01.619Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
85082024-02-15T03:01:01.619ZStates: Active Active Active
85092024-02-15T03:01:01.619ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85102024-02-15T03:01:01.620Z 1 Acked 1000 Write 7 Done Done Done false
85112024-02-15T03:01:01.623Z 2 Acked 1001 Write 1 Done Done Done false
85122024-02-15T03:01:01.623Z 3 Acked 1002 Write 5 Done Done Done false
85132024-02-15T03:01:01.623Z 4 Acked 1003 Write 9 Done Done Done false
85142024-02-15T03:01:01.623Z 5 Acked 1004 Read 7 Done Sent Done false
85152024-02-15T03:01:01.623Z 6 Acked 1005 Read 6 Done Sent Done false
85162024-02-15T03:01:01.623Z 7 Acked 1006 Read 5 Done Sent Done false
85172024-02-15T03:01:01.623Z 8 Acked 1007 Write 2 Done Sent Done false
85182024-02-15T03:01:01.623Z 9 Acked 1008 Flush 0 Done Sent Done false
85192024-02-15T03:01:01.624Z 10 Acked 1009 Flush 0 Done Sent Done false
85202024-02-15T03:01:01.624Z 11 Acked 1010 Write 2 Done Sent Done false
85212024-02-15T03:01:01.624Z 12 Acked 1011 Write 3 Done Sent Done false
85222024-02-15T03:01:01.624Z 13 Acked 1012 Flush 0 Done Sent Done false
85232024-02-15T03:01:01.624Z 14 Acked 1013 Write 8 Done Sent Done false
85242024-02-15T03:01:01.624Z 15 Acked 1014 Read 10 Done Sent Done false
85252024-02-15T03:01:01.624Z 16 Acked 1015 Write 4 Done Sent Done false
85262024-02-15T03:01:01.625Z 17 Acked 1016 Read 5 Done Sent Done false
85272024-02-15T03:01:01.625Z 18 Acked 1017 Write 6 Done Sent Done false
85282024-02-15T03:01:01.625Z 19 Acked 1018 Write 3 Done Sent Done false
85292024-02-15T03:01:01.625Z 20 Acked 1019 Write 1 Done Sent Done false
85302024-02-15T03:01:01.625Z 21 Acked 1020 Flush 0 Done Sent Done false
85312024-02-15T03:01:01.625Z 22 Acked 1021 Write 4 Done Sent Done false
85322024-02-15T03:01:01.625Z 23 Acked 1022 Read 9 Done Sent Done false
85332024-02-15T03:01:01.625Z 24 Acked 1023 Read 4 Done Sent Done false
85342024-02-15T03:01:01.625Z 25 Acked 1024 Write 3 Done Sent Done false
85352024-02-15T03:01:01.626Z 26 Acked 1025 Read 8 Done Sent Done false
85362024-02-15T03:01:01.626Z 27 Acked 1026 Write 9 Done Sent Done false
85372024-02-15T03:01:01.626Z 28 Acked 1027 Write 4 Done Sent Done false
85382024-02-15T03:01:01.626Z 29 Acked 1028 Read 3 Done Sent Done false
85392024-02-15T03:01:01.626Z 30 Acked 1029 Read 10 Done Sent Sent false
85402024-02-15T03:01:01.626Z STATES DS:0 DS:1 DS:2 TOTAL
85412024-02-15T03:01:01.626Z New 0 0 0 0
85422024-02-15T03:01:01.626Z Sent 0 26 1 27
85432024-02-15T03:01:01.626Z Done 30 4 29 63
85442024-02-15T03:01:01.626Z Skipped 0 0 0 0
85452024-02-15T03:01:01.627Z Error 0 0 0 0
85462024-02-15T03:01:01.627ZLast Flush: 1020 0 1020
85472024-02-15T03:01:01.627ZDownstairs last five completed:
85482024-02-15T03:01:01.627ZUpstairs last five completed: 30 29 28 27 26
85492024-02-15T03:01:01.627Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:00.514008954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1439}
85502024-02-15T03:01:01.627ZWrote out file "/tmp/test_repair_verify.data"
85512024-02-15T03:01:01.627Z
85522024-02-15T03:01:01.627ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85532024-02-15T03:01:01.652ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85542024-02-15T03:01:01.652Z 0 000-019 70 70 70 126 126 126 F F F
85552024-02-15T03:01:01.652Z 1 020-039 71 71 71 127 127 127 F F F
85562024-02-15T03:01:01.653Z 2 040-059 65 65 65 120 120 120 T F T <---
85572024-02-15T03:01:01.653Z 3 060-079 68 68 68 124 124 124 F F F
85582024-02-15T03:01:01.653Z 4 080-099 76 74 76 131 130 131 F T F <---
85592024-02-15T03:01:01.653Z 5 100-119 76 74 76 134 130 134 F F F <---
85602024-02-15T03:01:01.653Z 6 120-139 70 70 70 126 126 126 F F F
85612024-02-15T03:01:01.653Z 7 140-159 70 70 70 126 126 126 F F F
85622024-02-15T03:01:01.653Z 8 160-179 76 74 76 133 130 133 F F F <---
85632024-02-15T03:01:01.654Z 9 180-199 74 74 74 130 130 130 T F T <---
85642024-02-15T03:01:01.654Z 10 200-219 74 74 74 130 130 130 F F F
85652024-02-15T03:01:01.654Z 11 220-239 67 67 67 122 122 122 F F F
85662024-02-15T03:01:01.654Z 12 240-259 76 72 76 131 128 131 F F F <---
85672024-02-15T03:01:01.654Z 13 260-279 74 74 74 130 130 130 F F F
85682024-02-15T03:01:01.654Z 14 280-299 76 72 76 131 128 131 F T F <---
85692024-02-15T03:01:01.654Z 15 300-319 68 68 68 124 124 124 F F F
85702024-02-15T03:01:01.654Z 16 320-339 73 73 73 129 129 129 F F F
85712024-02-15T03:01:01.655Z 17 340-359 76 74 76 134 130 134 F F F <---
85722024-02-15T03:01:01.655Z 18 360-379 74 74 74 130 130 130 F F F
85732024-02-15T03:01:01.655Z 19 380-399 76 68 76 134 123 134 T F T <---
85742024-02-15T03:01:01.655Z 20 400-419 76 73 76 131 129 131 T T T <---
85752024-02-15T03:01:01.655Z 21 420-439 76 70 76 133 126 133 F T F <---
85762024-02-15T03:01:01.655Z 22 440-459 73 73 73 129 129 129 F F F
85772024-02-15T03:01:01.655Z 23 460-479 74 74 74 130 130 130 F F F
85782024-02-15T03:01:01.655Z 24 480-499 68 68 68 124 124 124 F F F
85792024-02-15T03:01:01.655Z 25 500-519 59 59 59 107 107 107 F F F
85802024-02-15T03:01:01.655Z 26 520-539 71 71 71 127 127 127 F F F
85812024-02-15T03:01:01.656Z 27 540-559 74 74 74 130 130 130 F F F
85822024-02-15T03:01:01.656Z 28 560-579 76 74 76 134 130 134 F F F <---
85832024-02-15T03:01:01.656Z 29 580-599 68 68 68 124 124 124 T F T <---
85842024-02-15T03:01:01.656ZMax gen: 76, Max flush: 134
85852024-02-15T03:01:01.656ZError: Difference in extent metadata found!
85862024-02-15T03:01:01.656ZA Difference in extent metadata is expected here
85872024-02-15T03:01:01.656ZOn loop 38
85882024-02-15T03:01:01.657Z
85892024-02-15T03:01:01.657ZVerifying data now
85902024-02-15T03:01:09.646ZNo change after verify
85912024-02-15T03:01:09.646ZLoop: 38 Downstairs dump after verify (and repair):
85922024-02-15T03:01:09.647ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85932024-02-15T03:01:09.647Z 0 000-019 70 70 70 126 126 126 F F F
85942024-02-15T03:01:09.647Z 1 020-039 71 71 71 127 127 127 F F F
85952024-02-15T03:01:09.647Z 2 040-059 77 77 77 135 135 135 F F F
85962024-02-15T03:01:09.647Z 3 060-079 68 68 68 124 124 124 F F F
85972024-02-15T03:01:09.647Z 4 080-099 76 76 76 131 131 131 F F F
85982024-02-15T03:01:09.647Z 5 100-119 76 76 76 134 134 134 F F F
85992024-02-15T03:01:09.648Z 6 120-139 70 70 70 126 126 126 F F F
86002024-02-15T03:01:09.648Z 7 140-159 70 70 70 126 126 126 F F F
86012024-02-15T03:01:09.648Z 8 160-179 76 76 76 133 133 133 F F F
86022024-02-15T03:01:09.648Z 9 180-199 77 77 77 135 135 135 F F F
86032024-02-15T03:01:09.648Z 10 200-219 74 74 74 130 130 130 F F F
86042024-02-15T03:01:09.648Z 11 220-239 67 67 67 122 122 122 F F F
86052024-02-15T03:01:09.648Z 12 240-259 76 76 76 131 131 131 F F F
86062024-02-15T03:01:09.649Z 13 260-279 74 74 74 130 130 130 F F F
86072024-02-15T03:01:09.649Z 14 280-299 76 76 76 131 131 131 F F F
86082024-02-15T03:01:09.649Z 15 300-319 68 68 68 124 124 124 F F F
86092024-02-15T03:01:09.649Z 16 320-339 73 73 73 129 129 129 F F F
86102024-02-15T03:01:09.649Z 17 340-359 76 76 76 134 134 134 F F F
86112024-02-15T03:01:09.649Z 18 360-379 74 74 74 130 130 130 F F F
86122024-02-15T03:01:09.649Z 19 380-399 77 77 77 135 135 135 F F F
86132024-02-15T03:01:09.649Z 20 400-419 77 77 77 135 135 135 F F F
86142024-02-15T03:01:09.649Z 21 420-439 76 76 76 133 133 133 F F F
86152024-02-15T03:01:09.649Z 22 440-459 73 73 73 129 129 129 F F F
86162024-02-15T03:01:09.649Z 23 460-479 74 74 74 130 130 130 F F F
86172024-02-15T03:01:09.649Z 24 480-499 68 68 68 124 124 124 F F F
86182024-02-15T03:01:09.649Z 25 500-519 59 59 59 107 107 107 F F F
86192024-02-15T03:01:09.649Z 26 520-539 71 71 71 127 127 127 F F F
86202024-02-15T03:01:09.650Z 27 540-559 74 74 74 130 130 130 F F F
86212024-02-15T03:01:09.702Z 28 560-579 76 76 76 134 134 134 F F F
86222024-02-15T03:01:09.702Z 29 580-599 77 77 77 135 135 135 F F F
86232024-02-15T03:01:09.702ZMax gen: 77, Max flush: 135
86242024-02-15T03:01:09.702Z
86252024-02-15T03:01:09.702ZBegin loop 39
86262024-02-15T03:01:09.702ZDownstairs to restart: 2
86272024-02-15T03:01:09.702Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249142539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447}
86282024-02-15T03:01:09.702Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
86292024-02-15T03:01:09.702Z,"time":"2024-02-15T03:01:06.249400106Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447}
86302024-02-15T03:01:09.702Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249438472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447}
86312024-02-15T03:01:09.702Z{"msg":"Crucible 6398ff4b-ae62-412a-85a7-cd80cd02fa1f has session id: 8f819d4f-a1cb-433a-946f-8b4f9a57ae63","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249463929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86322024-02-15T03:01:09.703Z{"msg":"Upstairs opts: Upstairs UUID: 6398ff4b-ae62-412a-85a7-cd80cd02fa1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249503993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86332024-02-15T03:01:09.703Z{"msg":"Crucible stats registered with UUID: 6398ff4b-ae62-412a-85a7-cd80cd02fa1f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.24952725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86342024-02-15T03:01:09.703Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249551947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447}
86352024-02-15T03:01:09.703Z{"msg":"6398ff4b-ae62-412a-85a7-cd80cd02fa1f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249649375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86362024-02-15T03:01:09.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249675202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86372024-02-15T03:01:09.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.24969217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86382024-02-15T03:01:09.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249714937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86392024-02-15T03:01:09.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249736714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86402024-02-15T03:01:09.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.24976919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86412024-02-15T03:01:09.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249786208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86422024-02-15T03:01:09.703Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249807915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86432024-02-15T03:01:09.703Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.24984919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86442024-02-15T03:01:09.703Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.249877737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86452024-02-15T03:01:09.703Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:06.249949937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86462024-02-15T03:01:09.703Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:06.249987623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86472024-02-15T03:01:09.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.250044806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86482024-02-15T03:01:09.703Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:06.250064503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86492024-02-15T03:01:09.703Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.25009334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86502024-02-15T03:01:09.703Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.250131235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86512024-02-15T03:01:09.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.250180019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86522024-02-15T03:01:09.704Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.250219934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86532024-02-15T03:01:09.704Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.25064545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86542024-02-15T03:01:09.704Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.250682476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86552024-02-15T03:01:09.704Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.25073333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86562024-02-15T03:01:09.704Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.250776334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86572024-02-15T03:01:09.704Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.25129003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86582024-02-15T03:01:09.704Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.251342983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86592024-02-15T03:01:09.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.251954886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86602024-02-15T03:01:09.704Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.252000581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86612024-02-15T03:01:09.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.252018438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86622024-02-15T03:01:09.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.252042895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86632024-02-15T03:01:09.704Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.252079861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86642024-02-15T03:01:09.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:06.252118396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86652024-02-15T03:01:09.704Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.251779511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86662024-02-15T03:01:09.704Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.251895697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86672024-02-15T03:01:09.704Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.252677919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86682024-02-15T03:01:09.704Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.252717034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86692024-02-15T03:01:09.704Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253201653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86702024-02-15T03:01:09.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253773232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86712024-02-15T03:01:09.704Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253817636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86722024-02-15T03:01:09.704Z{"msg":"[0]R flush_numbers[0..12]: [126, 127, 135, 124, 131, 134, 126, 126, 133, 135, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253858411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86732024-02-15T03:01:09.704Z{"msg":"[0]R generation[0..12]: [70, 71, 77, 68, 76, 76, 70, 70, 76, 77, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253894026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86742024-02-15T03:01:09.704Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253911494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86752024-02-15T03:01:09.704Z{"msg":"[1]R flush_numbers[0..12]: [126, 127, 135, 124, 131, 134, 126, 126, 133, 135, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253933521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86762024-02-15T03:01:09.704Z{"msg":"[1]R generation[0..12]: [70, 71, 77, 68, 76, 76, 70, 70, 76, 77, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.25394843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86772024-02-15T03:01:09.705ZWait for a query_work_queue command to finish before sending IO
86782024-02-15T03:01:09.705Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.253977506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86792024-02-15T03:01:09.705Z{"msg":"[2]R flush_numbers[0..12]: [126, 127, 135, 124, 131, 134, 126, 126, 133, 135, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254001213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86802024-02-15T03:01:09.705Z{"msg":"[2]R generation[0..12]: [70, 71, 77, 68, 76, 76, 70, 70, 76, 77, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254016811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86812024-02-15T03:01:09.705Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254037919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86822024-02-15T03:01:09.705Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254052527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86832024-02-15T03:01:09.705Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254073254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86842024-02-15T03:01:09.705Z{"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254087722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86852024-02-15T03:01:09.705Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254101651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86862024-02-15T03:01:09.705Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254121898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86872024-02-15T03:01:09.705Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254134636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86882024-02-15T03:01:09.705Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254155394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86892024-02-15T03:01:09.705Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254169492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86902024-02-15T03:01:09.705Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.2541899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86912024-02-15T03:01:09.705ZError: Difference in extent metadata found!
86922024-02-15T03:01:09.705Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254203878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86932024-02-15T03:01:09.705Z{"msg":"6398ff4b-ae62-412a-85a7-cd80cd02fa1f is now active with session: 8f819d4f-a1cb-433a-946f-8b4f9a57ae63","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254217406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86942024-02-15T03:01:09.705Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254229904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447,"session_id":"8f819d4f-a1cb-433a-946f-8b4f9a57ae63"}
86952024-02-15T03:01:09.705Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:07.254242563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447}
86962024-02-15T03:01:09.705ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86972024-02-15T03:01:09.705ZLoading write count information from file "/tmp/test_repair_verify.data"
86982024-02-15T03:01:09.705ZRun Repair workload
86992024-02-15T03:01:09.705Z01/30 Write block 591 len 20480 data: 10 8 9 10 8
87002024-02-15T03:01:09.705Z02/30 Write block 212 len 40960 data: 7 7 8 7 6 7 7 9 10 10
87012024-02-15T03:01:09.705Z03/30 Read block 555 len 12288
87022024-02-15T03:01:09.706Z04/30 Write block 405 len 36864 data: 7 8 9 9 12 11 10 10 10
87032024-02-15T03:01:09.706Z05/30 Read block 277 len 28672
87042024-02-15T03:01:09.706Z06/30 Read block 135 len 32768
87052024-02-15T03:01:09.706Z07/30 Read block 29 len 12288
87062024-02-15T03:01:09.706Z08/30 Read block 344 len 4096
87072024-02-15T03:01:09.706Z09/30 Read block 408 len 32768
87082024-02-15T03:01:09.706Z10/30 Write block 130 len 16384 data: 4 5 7 8
87092024-02-15T03:01:09.706Z11/30 Read block 2 len 24576
87102024-02-15T03:01:09.706Z12/30 Read block 444 len 24576
87112024-02-15T03:01:09.706Z13/30 Write block 244 len 20480 data: 7 8 8 6 6
87122024-02-15T03:01:09.706Z14/30 Read block 385 len 20480
87132024-02-15T03:01:09.706Z15/30 Read block 469 len 8192
87142024-02-15T03:01:09.706Z16/30 Read block 585 len 12288
87152024-02-15T03:01:09.706Z17/30 Write block 149 len 40960 data: 6 5 5 5 7 6 6 10 10 9
87162024-02-15T03:01:09.706Z18/30 Read block 557 len 16384
87172024-02-15T03:01:09.706Z19/30 Write block 545 len 12288 data: 4 5 5
87182024-02-15T03:01:09.706Z20/30 Write block 178 len 28672 data: 7 7 7 6 5 7 8
87192024-02-15T03:01:09.706Z21/30 Read block 533 len 12288
87202024-02-15T03:01:09.706Z22/30 Write block 458 len 4096 data: 3
87212024-02-15T03:01:09.721Z23/30 Read block 199 len 20480
87222024-02-15T03:01:09.721Z24/30 Read block 593 len 4096
87232024-02-15T03:01:09.721Z25/30 Write block 3 len 16384 data: 4 6 6 7
87242024-02-15T03:01:09.721Z26/30 Read block 307 len 40960
87252024-02-15T03:01:09.722Z27/30 Read block 410 len 28672
87262024-02-15T03:01:09.722Z28/30 Write block 358 len 40960 data: 8 8 7 6 7 5 6 6 7 7
87272024-02-15T03:01:09.722Z29/30 Write block 408 len 36864 data: 10 13 12 11 11 11 9 7 7
87282024-02-15T03:01:09.722Z30/30 Write block 478 len 12288 data: 7 7 6
87292024-02-15T03:01:09.722Z----------------------------------------------------------------
87302024-02-15T03:01:09.722Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
87312024-02-15T03:01:09.722ZStates: Active Active Active
87322024-02-15T03:01:09.722ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87332024-02-15T03:01:09.722Z 1 Acked 1000 Write 5 Done Done Done false
87342024-02-15T03:01:09.722Z 2 Acked 1001 Write 10 Done Done Sent false
87352024-02-15T03:01:09.722Z 3 Acked 1002 Read 3 Done Done Sent false
87362024-02-15T03:01:09.722Z 4 Acked 1003 Write 9 Done Done Sent false
87372024-02-15T03:01:09.723Z 5 Acked 1004 Read 7 Done Done Sent false
87382024-02-15T03:01:09.723Z 6 Acked 1005 Read 8 Done Done Sent false
87392024-02-15T03:01:09.723Z 7 Acked 1006 Read 3 Done Done Sent false
87402024-02-15T03:01:09.723Z 8 Acked 1007 Read 1 Done Done Sent false
87412024-02-15T03:01:09.723Z 9 Acked 1008 Read 8 Done Done Sent false
87422024-02-15T03:01:09.723Z 10 Acked 1009 Write 4 Done Done Sent false
87432024-02-15T03:01:09.723Z 11 Acked 1010 Read 6 Done Done Sent false
87442024-02-15T03:01:09.723Z 12 Acked 1011 Read 6 Done Done Sent false
87452024-02-15T03:01:09.723Z 13 Acked 1012 Write 5 Done Done Sent false
87462024-02-15T03:01:09.723Z 14 Acked 1013 Read 5 Done Done Sent false
87472024-02-15T03:01:09.723Z 15 Acked 1014 Read 2 Done Done Sent false
87482024-02-15T03:01:09.724Z 16 Acked 1015 Read 3 Done Done Sent false
87492024-02-15T03:01:09.724Z 17 Acked 1016 Write 10 Done Done Sent false
87502024-02-15T03:01:09.724Z 18 Acked 1017 Read 4 Done Done Sent false
87512024-02-15T03:01:09.724Z 19 Acked 1018 Write 3 Done Done Sent false
87522024-02-15T03:01:09.724Z 20 Acked 1019 Write 7 Done Done Sent false
87532024-02-15T03:01:09.724Z 21 Acked 1020 Read 3 Done Done Sent false
87542024-02-15T03:01:09.724Z 22 Acked 1021 Write 1 Done Done Sent false
87552024-02-15T03:01:09.724Z 23 Acked 1022 Read 5 Done Done Sent false
87562024-02-15T03:01:09.724Z 24 Acked 1023 Read 1 Done Done Sent false
87572024-02-15T03:01:09.724Z 25 Acked 1024 Write 4 Done Done Sent false
87582024-02-15T03:01:09.724Z 26 Acked 1025 Read 10 Done Done Sent false
87592024-02-15T03:01:09.725Z 27 Acked 1026 Read 7 Done Done Sent false
87602024-02-15T03:01:09.725Z 28 Acked 1027 Write 10 Done Done Sent false
87612024-02-15T03:01:09.725Z 29 Acked 1028 Write 9 Sent Sent Sent false
87622024-02-15T03:01:09.725Z 30 Acked 1029 Write 3 Sent Sent Sent false
87632024-02-15T03:01:09.725Z STATES DS:0 DS:1 DS:2 TOTAL
87642024-02-15T03:01:09.725Z New 0 0 0 0
87652024-02-15T03:01:09.725Z Sent 2 2 29 33
87662024-02-15T03:01:09.725Z Done 28 28 1 57
87672024-02-15T03:01:09.725Z Skipped 0 0 0 0
87682024-02-15T03:01:09.725Z Error 0 0 0 0
87692024-02-15T03:01:09.725ZLast Flush: 0 0 0
87702024-02-15T03:01:09.725ZDownstairs last five completed:
87712024-02-15T03:01:09.725ZUpstairs last five completed: 30 29 28 27 26
87722024-02-15T03:01:09.725Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:07.281852063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1447}
87732024-02-15T03:01:09.726ZWrote out file "/tmp/test_repair_verify.data"
87742024-02-15T03:01:09.726Z
87752024-02-15T03:01:09.726ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87762024-02-15T03:01:09.726ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87772024-02-15T03:01:09.726Z 0 000-019 70 70 70 126 126 126 T T F <---
87782024-02-15T03:01:09.726Z 1 020-039 71 71 71 127 127 127 F F F
87792024-02-15T03:01:09.726Z 2 040-059 77 77 77 135 135 135 F F F
87802024-02-15T03:01:09.726Z 3 060-079 68 68 68 124 124 124 F F F
87812024-02-15T03:01:09.726Z 4 080-099 76 76 76 131 131 131 F F F
87822024-02-15T03:01:09.726Z 5 100-119 76 76 76 134 134 134 F F F
87832024-02-15T03:01:09.726Z 6 120-139 70 70 70 126 126 126 T T F <---
87842024-02-15T03:01:09.726Z 7 140-159 70 70 70 126 126 126 T T F <---
87852024-02-15T03:01:09.726Z 8 160-179 76 76 76 133 133 133 T T F <---
87862024-02-15T03:01:09.727Z 9 180-199 77 77 77 135 135 135 T T F <---
87872024-02-15T03:01:09.727Z 10 200-219 74 74 74 130 130 130 T T F <---
87882024-02-15T03:01:09.727Z 11 220-239 67 67 67 122 122 122 T T F <---
87892024-02-15T03:01:09.727Z 12 240-259 76 76 76 131 131 131 T T F <---
87902024-02-15T03:01:09.727Z 13 260-279 74 74 74 130 130 130 F F F
87912024-02-15T03:01:09.727Z 14 280-299 76 76 76 131 131 131 F F F
87922024-02-15T03:01:09.727Z 15 300-319 68 68 68 124 124 124 F F F
87932024-02-15T03:01:09.727Z 16 320-339 73 73 73 129 129 129 F F F
87942024-02-15T03:01:09.727Z 17 340-359 76 76 76 134 134 134 T T F <---
87952024-02-15T03:01:09.727Z 18 360-379 74 74 74 130 130 130 T T F <---
87962024-02-15T03:01:09.727Z 19 380-399 77 77 77 135 135 135 F F F
87972024-02-15T03:01:09.727Z 20 400-419 77 77 77 135 135 135 T T F <---
87982024-02-15T03:01:09.727Z 21 420-439 76 76 76 133 133 133 F F F
87992024-02-15T03:01:09.728Z 22 440-459 73 73 73 129 129 129 T T F <---
88002024-02-15T03:01:09.728Z 23 460-479 74 74 74 130 130 130 T T F <---
88012024-02-15T03:01:09.728Z 24 480-499 68 68 68 124 124 124 T T F <---
88022024-02-15T03:01:09.728Z 25 500-519 59 59 59 107 107 107 F F F
88032024-02-15T03:01:09.728Z 26 520-539 71 71 71 127 127 127 F F F
88042024-02-15T03:01:09.728Z 27 540-559 74 74 74 130 130 130 T T F <---
88052024-02-15T03:01:09.728Z 28 560-579 76 76 76 134 134 134 F F F
88062024-02-15T03:01:09.728Z 29 580-599 77 77 77 135 135 135 T T T
88072024-02-15T03:01:09.728ZMax gen: 77, Max flush: 135
88082024-02-15T03:01:09.728ZA Difference in extent metadata is expected here
88092024-02-15T03:01:09.728ZOn loop 39
88102024-02-15T03:01:09.728Z
88112024-02-15T03:01:09.728ZVerifying data now
88122024-02-15T03:01:14.195ZNo change after verify
88132024-02-15T03:01:14.195ZLoop: 39 Downstairs dump after verify (and repair):
88142024-02-15T03:01:14.210ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88152024-02-15T03:01:14.211Z 0 000-019 78 78 78 136 136 136 F F F
88162024-02-15T03:01:14.211Z 1 020-039 71 71 71 127 127 127 F F F
88172024-02-15T03:01:14.211Z 2 040-059 77 77 77 135 135 135 F F F
88182024-02-15T03:01:14.211Z 3 060-079 68 68 68 124 124 124 F F F
88192024-02-15T03:01:14.211Z 4 080-099 76 76 76 131 131 131 F F F
88202024-02-15T03:01:14.211Z 5 100-119 76 76 76 134 134 134 F F F
88212024-02-15T03:01:14.211Z 6 120-139 78 78 78 136 136 136 F F F
88222024-02-15T03:01:14.211Z 7 140-159 78 78 78 136 136 136 F F F
88232024-02-15T03:01:14.211Z 8 160-179 78 78 78 136 136 136 F F F
88242024-02-15T03:01:14.211Z 9 180-199 78 78 78 136 136 136 F F F
88252024-02-15T03:01:14.211Z 10 200-219 78 78 78 136 136 136 F F F
88262024-02-15T03:01:14.215Z 11 220-239 78 78 78 136 136 136 F F F
88272024-02-15T03:01:14.215Z 12 240-259 78 78 78 136 136 136 F F F
88282024-02-15T03:01:14.215Z 13 260-279 74 74 74 130 130 130 F F F
88292024-02-15T03:01:14.215Z 14 280-299 76 76 76 131 131 131 F F F
88302024-02-15T03:01:14.215Z 15 300-319 68 68 68 124 124 124 F F F
88312024-02-15T03:01:14.215Z 16 320-339 73 73 73 129 129 129 F F F
88322024-02-15T03:01:14.215Z 17 340-359 78 78 78 136 136 136 F F F
88332024-02-15T03:01:14.215Z 18 360-379 78 78 78 136 136 136 F F F
88342024-02-15T03:01:14.215Z 19 380-399 77 77 77 135 135 135 F F F
88352024-02-15T03:01:14.215Z 20 400-419 78 78 78 136 136 136 F F F
88362024-02-15T03:01:14.215Z 21 420-439 76 76 76 133 133 133 F F F
88372024-02-15T03:01:14.215Z 22 440-459 78 78 78 136 136 136 F F F
88382024-02-15T03:01:14.215Z 23 460-479 78 78 78 136 136 136 F F F
88392024-02-15T03:01:14.215Z 24 480-499 78 78 78 136 136 136 F F F
88402024-02-15T03:01:14.215Z 25 500-519 59 59 59 107 107 107 F F F
88412024-02-15T03:01:14.215Z 26 520-539 71 71 71 127 127 127 F F F
88422024-02-15T03:01:14.215Z 27 540-559 78 78 78 136 136 136 F F F
88432024-02-15T03:01:14.215Z 28 560-579 76 76 76 134 134 134 F F F
88442024-02-15T03:01:14.215Z 29 580-599 78 78 78 136 136 136 F F F
88452024-02-15T03:01:14.215ZMax gen: 78, Max flush: 136
88462024-02-15T03:01:14.215Z
88472024-02-15T03:01:14.216ZBegin loop 40
88482024-02-15T03:01:14.216ZDownstairs to restart: 2
88492024-02-15T03:01:14.235Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.159414133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455}
88502024-02-15T03:01:14.235Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.15975549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455}
88512024-02-15T03:01:14.235Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
88522024-02-15T03:01:14.240Z,"time":"2024-02-15T03:01:13.159802354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455}
88532024-02-15T03:01:14.240Z{"msg":"Crucible 72eb4308-f1d4-43ea-8306-b51477b64908 has session id: 97822958-1cc7-449c-9c49-65eec2854315","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.159875575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88542024-02-15T03:01:14.240Z{"msg":"Upstairs opts: Upstairs UUID: 72eb4308-f1d4-43ea-8306-b51477b64908, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.159971053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88552024-02-15T03:01:14.240Z{"msg":"Crucible stats registered with UUID: 72eb4308-f1d4-43ea-8306-b51477b64908","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160019217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88562024-02-15T03:01:14.240Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160046693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455}
88572024-02-15T03:01:14.240Z{"msg":"72eb4308-f1d4-43ea-8306-b51477b64908 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.16007549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88582024-02-15T03:01:14.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160098287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88592024-02-15T03:01:14.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160133692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88602024-02-15T03:01:14.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160157309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88612024-02-15T03:01:14.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160182446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88622024-02-15T03:01:14.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160218742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88632024-02-15T03:01:14.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160257537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88642024-02-15T03:01:14.241Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160296342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88652024-02-15T03:01:14.241Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160364674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88662024-02-15T03:01:14.241Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160422636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88672024-02-15T03:01:14.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.16047282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88682024-02-15T03:01:14.241Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:13.160509695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88692024-02-15T03:01:14.241Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:13.160556219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88702024-02-15T03:01:14.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160640389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88712024-02-15T03:01:14.241Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:13.160701551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88722024-02-15T03:01:14.241Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160718759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88732024-02-15T03:01:14.241Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160733357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88742024-02-15T03:01:14.241Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.160911495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88752024-02-15T03:01:14.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.161320034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88762024-02-15T03:01:14.241Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.161364568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88772024-02-15T03:01:14.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.161393114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88782024-02-15T03:01:14.241Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.161418671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88792024-02-15T03:01:14.242Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.161996789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88802024-02-15T03:01:14.242Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.162054262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88812024-02-15T03:01:14.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.162552939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88822024-02-15T03:01:14.242Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.162589984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88832024-02-15T03:01:14.242Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.162605283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88842024-02-15T03:01:14.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.162631409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88852024-02-15T03:01:14.242Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.162652717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88862024-02-15T03:01:14.242Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:13.162666625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88872024-02-15T03:01:15.237Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.161597682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88882024-02-15T03:01:15.237Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.161672172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88892024-02-15T03:01:15.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.162522916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88902024-02-15T03:01:15.243Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.162559851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88912024-02-15T03:01:15.243Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.163216779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88922024-02-15T03:01:15.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.163813374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88932024-02-15T03:01:15.243Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.163846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88942024-02-15T03:01:15.243Z{"msg":"[0]R flush_numbers[0..12]: [136, 127, 135, 124, 131, 134, 136, 136, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.163868847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88952024-02-15T03:01:15.243Z{"msg":"[0]R generation[0..12]: [78, 71, 77, 68, 76, 76, 78, 78, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.163900133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88962024-02-15T03:01:15.243Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.163946407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88972024-02-15T03:01:15.243Z{"msg":"[1]R flush_numbers[0..12]: [136, 127, 135, 124, 131, 134, 136, 136, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.163982533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88982024-02-15T03:01:15.243Z{"msg":"[1]R generation[0..12]: [78, 71, 77, 68, 76, 76, 78, 78, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.16400137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
88992024-02-15T03:01:15.243Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164035556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89002024-02-15T03:01:15.243ZWait for a query_work_queue command to finish before sending IO
89012024-02-15T03:01:15.243Z{"msg":"[2]R flush_numbers[0..12]: [136, 127, 135, 124, 131, 134, 136, 136, 136, 136, 136, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.1640857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89022024-02-15T03:01:15.243Z{"msg":"[2]R generation[0..12]: [78, 71, 77, 68, 76, 76, 78, 78, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164131634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89032024-02-15T03:01:15.243Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.16416294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89042024-02-15T03:01:15.243Z{"msg":"Max found gen is 79","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164181228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89052024-02-15T03:01:15.243Z{"msg":"Generation requested: 80 >= found:79","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164222733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89062024-02-15T03:01:15.243Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164267077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89072024-02-15T03:01:15.243Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164298713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89082024-02-15T03:01:15.243Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164338548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89092024-02-15T03:01:15.243Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164381523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89102024-02-15T03:01:15.243Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164414559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89112024-02-15T03:01:15.243Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164451304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89122024-02-15T03:01:15.244Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.16448834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89132024-02-15T03:01:15.244Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164504558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89142024-02-15T03:01:15.244Z{"msg":"72eb4308-f1d4-43ea-8306-b51477b64908 is now active with session: 97822958-1cc7-449c-9c49-65eec2854315","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164530274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89152024-02-15T03:01:15.244Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.1645646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455,"session_id":"97822958-1cc7-449c-9c49-65eec2854315"}
89162024-02-15T03:01:15.244Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:14.164593996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455}
89172024-02-15T03:01:15.244ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
89182024-02-15T03:01:15.244ZLoading write count information from file "/tmp/test_repair_verify.data"
89192024-02-15T03:01:15.244ZRun Repair workload
89202024-02-15T03:01:15.244Z01/30 Write block 410 len 8192 data: 13 12
89212024-02-15T03:01:15.247Z02/30 Flush
89222024-02-15T03:01:15.247Z03/30 Write block 454 len 32768 data: 3 4 4 4 4 4 8 7
89232024-02-15T03:01:15.247Z04/30 Read block 411 len 32768
89242024-02-15T03:01:15.247Z05/30 Read block 554 len 8192
89252024-02-15T03:01:15.247Z06/30 Write block 494 len 24576 data: 8 8 8 8 8 7
89262024-02-15T03:01:15.256Z07/30 Write block 17 len 20480 data: 3 5 4 4 3
89272024-02-15T03:01:15.256Z08/30 Write block 476 len 28672 data: 5 7 8 8 7 7 7
89282024-02-15T03:01:15.256Z09/30 Read block 441 len 40960
89292024-02-15T03:01:15.256Z10/30 Write block 530 len 24576 data: 6 4 7 5 5 6
89302024-02-15T03:01:15.256Z11/30 Write block 201 len 16384 data: 7 5 5 5
89312024-02-15T03:01:15.256Z12/30 Read block 323 len 12288
89322024-02-15T03:01:15.256Z13/30 Read block 340 len 8192
89332024-02-15T03:01:15.256Z14/30 Read block 466 len 24576
89342024-02-15T03:01:15.256Z15/30 Read block 367 len 36864
89352024-02-15T03:01:15.259Z16/30 Read block 506 len 32768
89362024-02-15T03:01:15.259Z17/30 Read block 118 len 28672
89372024-02-15T03:01:15.259Z18/30 Write block 512 len 16384 data: 10 10 10 7
89382024-02-15T03:01:15.259Z19/30 Read block 99 len 32768
89392024-02-15T03:01:15.262Z20/30 Read block 249 len 12288
89402024-02-15T03:01:15.262Z21/30 Read block 311 len 16384
89412024-02-15T03:01:15.262Z22/30 Write block 154 len 24576 data: 7 7 11 11 10 7
89422024-02-15T03:01:15.265Z23/30 Read block 432 len 4096
89432024-02-15T03:01:15.265Z24/30 Read block 509 len 20480
89442024-02-15T03:01:15.265Z25/30 Flush
89452024-02-15T03:01:15.269Z26/30 Write block 495 len 20480 data: 9 9 9 9 8
89462024-02-15T03:01:15.282Z27/30 Read block 597 len 12288
89472024-02-15T03:01:15.282Z28/30 Write block 228 len 28672 data: 8 6 5 6 7 7 8
89482024-02-15T03:01:15.282Z29/30 Write block 450 len 40960 data: 6 5 4 3 4 5 5 5 5 5
89492024-02-15T03:01:15.283Z30/30 Read block 216 len 28672
89502024-02-15T03:01:15.283Z----------------------------------------------------------------
89512024-02-15T03:01:15.283Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
89522024-02-15T03:01:15.283ZStates: Active Active Active
89532024-02-15T03:01:15.283ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89542024-02-15T03:01:15.283Z 1 Acked 1000 Write 2 Done Done Sent false
89552024-02-15T03:01:15.284Z 2 Acked 1001 Flush 0 Done Done Sent false
89562024-02-15T03:01:15.284Z 3 Acked 1002 Write 8 Done Done Sent false
89572024-02-15T03:01:15.284Z 4 Acked 1003 Read 8 Done Done Sent false
89582024-02-15T03:01:15.284Z 5 Acked 1004 Read 2 Done Done Sent false
89592024-02-15T03:01:15.284Z 6 Acked 1005 Write 6 Done Done Sent false
89602024-02-15T03:01:15.285Z 7 Acked 1006 Write 5 Done Done Sent false
89612024-02-15T03:01:15.285Z 8 Acked 1007 Write 7 Done Done Sent false
89622024-02-15T03:01:15.285Z 9 Acked 1008 Read 10 Done Done Sent false
89632024-02-15T03:01:15.285Z 10 Acked 1009 Write 6 Done Done Sent false
89642024-02-15T03:01:15.285Z 11 Acked 1010 Write 4 Done Done Sent false
89652024-02-15T03:01:15.286Z 12 Acked 1011 Read 3 Done Done Sent false
89662024-02-15T03:01:15.286Z 13 Acked 1012 Read 2 Done Done Sent false
89672024-02-15T03:01:15.286Z 14 Acked 1013 Read 6 Done Done Sent false
89682024-02-15T03:01:15.286Z 15 Acked 1014 Read 9 Done Done Sent false
89692024-02-15T03:01:15.286Z 16 Acked 1015 Read 8 Done Done Sent false
89702024-02-15T03:01:15.286Z 17 Acked 1016 Read 7 Done Done Sent false
89712024-02-15T03:01:15.286Z 18 Acked 1017 Write 4 Done Done Sent false
89722024-02-15T03:01:15.287Z 19 Acked 1018 Read 8 Done Done Sent false
89732024-02-15T03:01:15.287Z 20 Acked 1019 Read 3 Done Done Sent false
89742024-02-15T03:01:15.287Z 21 Acked 1020 Read 4 Done Done Sent false
89752024-02-15T03:01:15.287Z 22 Acked 1021 Write 6 Done Done Sent false
89762024-02-15T03:01:15.287Z 23 Acked 1022 Read 1 Done Done Sent false
89772024-02-15T03:01:15.287Z 24 Acked 1023 Read 5 Done Done Sent false
89782024-02-15T03:01:15.287Z 25 Acked 1024 Flush 0 Done Done Sent false
89792024-02-15T03:01:15.288Z 26 Acked 1025 Write 5 Done Done Sent false
89802024-02-15T03:01:15.288Z 27 Acked 1026 Read 3 Done Done Sent false
89812024-02-15T03:01:15.288Z 28 Acked 1027 Write 7 Done Done Sent false
89822024-02-15T03:01:15.288Z 29 Acked 1028 Write 10 Done Done Sent false
89832024-02-15T03:01:15.288Z 30 Acked 1029 Read 7 Done Sent Sent false
89842024-02-15T03:01:15.289Z STATES DS:0 DS:1 DS:2 TOTAL
89852024-02-15T03:01:15.289Z New 0 0 0 0
89862024-02-15T03:01:15.289Z Sent 0 1 30 31
89872024-02-15T03:01:15.289Z Done 30 29 0 59
89882024-02-15T03:01:15.289Z Skipped 0 0 0 0
89892024-02-15T03:01:15.290Z Error 0 0 0 0
89902024-02-15T03:01:15.290ZLast Flush: 1024 1024 0
89912024-02-15T03:01:15.290ZDownstairs last five completed:
89922024-02-15T03:01:15.290ZUpstairs last five completed: 30 29 28 27 26
89932024-02-15T03:01:15.290Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:14.198056103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1455}
89942024-02-15T03:01:15.290ZWrote out file "/tmp/test_repair_verify.data"
89952024-02-15T03:01:15.303Z
89962024-02-15T03:01:15.304ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89972024-02-15T03:01:15.313ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89982024-02-15T03:01:15.314Z 0 000-019 80 80 78 138 138 136 F F F <---
89992024-02-15T03:01:15.314Z 1 020-039 80 80 71 138 138 127 F F F <---
90002024-02-15T03:01:15.314Z 2 040-059 77 77 77 135 135 135 F F F
90012024-02-15T03:01:15.314Z 3 060-079 68 68 68 124 124 124 F F F
90022024-02-15T03:01:15.314Z 4 080-099 76 76 76 131 131 131 F F F
90032024-02-15T03:01:15.314Z 5 100-119 76 76 76 134 134 134 F F F
90042024-02-15T03:01:15.314Z 6 120-139 78 78 78 136 136 136 F F F
90052024-02-15T03:01:15.314Z 7 140-159 80 80 78 138 138 136 F F F <---
90062024-02-15T03:01:15.314Z 8 160-179 78 78 78 136 136 136 F F F
90072024-02-15T03:01:15.314Z 9 180-199 78 78 78 136 136 136 F F F
90082024-02-15T03:01:15.314Z 10 200-219 80 80 78 138 138 136 F F F <---
90092024-02-15T03:01:15.314Z 11 220-239 78 78 78 136 136 136 T T F <---
90102024-02-15T03:01:15.314Z 12 240-259 78 78 78 136 136 136 F F F
90112024-02-15T03:01:15.314Z 13 260-279 74 74 74 130 130 130 F F F
90122024-02-15T03:01:15.314Z 14 280-299 76 76 76 131 131 131 F F F
90132024-02-15T03:01:15.314Z 15 300-319 68 68 68 124 124 124 F F F
90142024-02-15T03:01:15.314Z 16 320-339 73 73 73 129 129 129 F F F
90152024-02-15T03:01:15.314Z 17 340-359 78 78 78 136 136 136 F F F
90162024-02-15T03:01:15.314Z 18 360-379 78 78 78 136 136 136 F F F
90172024-02-15T03:01:15.314Z 19 380-399 77 77 77 135 135 135 F F F
90182024-02-15T03:01:15.314Z 20 400-419 80 80 78 137 137 136 F F F <---
90192024-02-15T03:01:15.314Z 21 420-439 76 76 76 133 133 133 F F F
90202024-02-15T03:01:15.314Z 22 440-459 80 80 78 138 138 136 T T F <---
90212024-02-15T03:01:15.314Z 23 460-479 80 80 78 138 138 136 F F F <---
90222024-02-15T03:01:15.315Z 24 480-499 80 80 78 138 138 136 T T F <---
90232024-02-15T03:01:15.315Z 25 500-519 80 80 59 138 138 107 F F F <---
90242024-02-15T03:01:15.315Z 26 520-539 80 80 71 138 138 127 F F F <---
90252024-02-15T03:01:15.315Z 27 540-559 78 78 78 136 136 136 F F F
90262024-02-15T03:01:15.315Z 28 560-579 76 76 76 134 134 134 F F F
90272024-02-15T03:01:15.315Z 29 580-599 78 78 78 136 136 136 F F F
90282024-02-15T03:01:15.315ZMax gen: 80, Max flush: 138
90292024-02-15T03:01:15.315ZError: Difference in extent metadata found!
90302024-02-15T03:01:15.315ZA Difference in extent metadata is expected here
90312024-02-15T03:01:15.315ZOn loop 40
90322024-02-15T03:01:15.315Z
90332024-02-15T03:01:15.315ZVerifying data now
90342024-02-15T03:01:19.884ZNo change after verify
90352024-02-15T03:01:19.884ZLoop: 40 Downstairs dump after verify (and repair):
90362024-02-15T03:01:19.895ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90372024-02-15T03:01:19.896Z 0 000-019 80 80 80 138 138 138 F F F
90382024-02-15T03:01:19.896Z 1 020-039 80 80 80 138 138 138 F F F
90392024-02-15T03:01:19.896Z 2 040-059 77 77 77 135 135 135 F F F
90402024-02-15T03:01:19.896Z 3 060-079 68 68 68 124 124 124 F F F
90412024-02-15T03:01:19.896Z 4 080-099 76 76 76 131 131 131 F F F
90422024-02-15T03:01:19.897Z 5 100-119 76 76 76 134 134 134 F F F
90432024-02-15T03:01:19.897Z 6 120-139 78 78 78 136 136 136 F F F
90442024-02-15T03:01:19.897Z 7 140-159 80 80 80 138 138 138 F F F
90452024-02-15T03:01:19.903Z 8 160-179 78 78 78 136 136 136 F F F
90462024-02-15T03:01:19.903Z 9 180-199 78 78 78 136 136 136 F F F
90472024-02-15T03:01:19.903Z 10 200-219 80 80 80 138 138 138 F F F
90482024-02-15T03:01:19.904Z 11 220-239 81 81 81 139 139 139 F F F
90492024-02-15T03:01:19.904Z 12 240-259 78 78 78 136 136 136 F F F
90502024-02-15T03:01:19.904Z 13 260-279 74 74 74 130 130 130 F F F
90512024-02-15T03:01:19.904Z 14 280-299 76 76 76 131 131 131 F F F
90522024-02-15T03:01:19.904Z 15 300-319 68 68 68 124 124 124 F F F
90532024-02-15T03:01:19.904Z 16 320-339 73 73 73 129 129 129 F F F
90542024-02-15T03:01:19.904Z 17 340-359 78 78 78 136 136 136 F F F
90552024-02-15T03:01:19.904Z 18 360-379 78 78 78 136 136 136 F F F
90562024-02-15T03:01:19.904Z 19 380-399 77 77 77 135 135 135 F F F
90572024-02-15T03:01:19.905Z 20 400-419 80 80 80 137 137 137 F F F
90582024-02-15T03:01:19.905Z 21 420-439 76 76 76 133 133 133 F F F
90592024-02-15T03:01:19.905Z 22 440-459 81 81 81 139 139 139 F F F
90602024-02-15T03:01:19.905Z 23 460-479 80 80 80 138 138 138 F F F
90612024-02-15T03:01:19.905Z 24 480-499 81 81 81 139 139 139 F F F
90622024-02-15T03:01:19.905Z 25 500-519 80 80 80 138 138 138 F F F
90632024-02-15T03:01:19.905Z 26 520-539 80 80 80 138 138 138 F F F
90642024-02-15T03:01:19.905Z 27 540-559 78 78 78 136 136 136 F F F
90652024-02-15T03:01:19.905Z 28 560-579 76 76 76 134 134 134 F F F
90662024-02-15T03:01:19.905Z 29 580-599 78 78 78 136 136 136 F F F
90672024-02-15T03:01:19.905ZMax gen: 81, Max flush: 139
90682024-02-15T03:01:19.906Z
90692024-02-15T03:01:19.906ZBegin loop 41
90702024-02-15T03:01:19.906ZDownstairs to restart: 1
90712024-02-15T03:01:19.923Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.840478612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463}
90722024-02-15T03:01:19.923Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.840747508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463}
90732024-02-15T03:01:19.923Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.840766586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463}
90742024-02-15T03:01:19.923Z{"msg":"Crucible 74d3ad16-9266-47fd-8595-f27d58a43445 has session id: 22289d9d-0e17-4c33-b432-3a8d5c05416c","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.840781384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90752024-02-15T03:01:19.946ZCrucible runtime is spawned
90762024-02-15T03:01:19.946Z{"msg":"Upstairs opts: Upstairs UUID: 74d3ad16-9266-47fd-8595-f27d58a43445, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.840925016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90772024-02-15T03:01:19.946Z{"msg":"Crucible stats registered with UUID: 74d3ad16-9266-47fd-8595-f27d58a43445","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.840987878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90782024-02-15T03:01:19.946Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841122171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463}
90792024-02-15T03:01:19.946Z{"msg":"74d3ad16-9266-47fd-8595-f27d58a43445 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.84120815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90802024-02-15T03:01:19.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841272932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90812024-02-15T03:01:19.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841312597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90822024-02-15T03:01:19.947Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841365211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90832024-02-15T03:01:19.947Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841394777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90842024-02-15T03:01:19.947Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841440031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90852024-02-15T03:01:19.947Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841472387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90862024-02-15T03:01:19.947Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841513692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90872024-02-15T03:01:19.947Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841547518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90882024-02-15T03:01:19.947Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841591672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90892024-02-15T03:01:19.947Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:18.841627868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90902024-02-15T03:01:19.947Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:18.84168849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90912024-02-15T03:01:19.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841735844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90922024-02-15T03:01:19.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841775589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90932024-02-15T03:01:19.947Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:18.841819034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90942024-02-15T03:01:19.947Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.84185151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90952024-02-15T03:01:19.947Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841884896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90962024-02-15T03:01:19.948Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.841918041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90972024-02-15T03:01:19.948Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.84240668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90982024-02-15T03:01:19.948Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.842447755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
90992024-02-15T03:01:19.948Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.842476082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91002024-02-15T03:01:19.948Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.842503548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91012024-02-15T03:01:19.948Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.84304992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91022024-02-15T03:01:19.948Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.843087985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91032024-02-15T03:01:19.948Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.843733864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91042024-02-15T03:01:19.948Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.843782788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91052024-02-15T03:01:19.948Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.843801276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91062024-02-15T03:01:19.948Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.843823023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91072024-02-15T03:01:19.948Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.843837411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91082024-02-15T03:01:19.948Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:18.843863028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91092024-02-15T03:01:20.921Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.843531413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91102024-02-15T03:01:20.921Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.843623581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91112024-02-15T03:01:20.932Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.844502131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91122024-02-15T03:01:20.932Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.844540586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91132024-02-15T03:01:20.932Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.84507128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91142024-02-15T03:01:20.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845668035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91152024-02-15T03:01:20.932Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845719299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91162024-02-15T03:01:20.932Z{"msg":"[0]R flush_numbers[0..12]: [138, 138, 135, 124, 131, 134, 136, 138, 136, 136, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845749235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91172024-02-15T03:01:20.932Z{"msg":"[0]R generation[0..12]: [80, 80, 77, 68, 76, 76, 78, 80, 78, 78, 80, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845773922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91182024-02-15T03:01:20.932Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845800188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91192024-02-15T03:01:20.932Z{"msg":"[1]R flush_numbers[0..12]: [138, 138, 135, 124, 131, 134, 136, 138, 136, 136, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845836154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"Wait for a query_work_queue command to finish before sending IO
91202024-02-15T03:01:20.932Z:1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91212024-02-15T03:01:20.932Z{"msg":"[1]R generation[0..12]: [80, 80, 77, 68, 76, 76, 78, 80, 78, 78, 80, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845899876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91222024-02-15T03:01:20.932Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845928322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91232024-02-15T03:01:20.932Z{"msg":"[2]R flush_numbers[0..12]: [138, 138, 135, 124, 131, 134, 136, 138, 136, 136, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845958689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91242024-02-15T03:01:20.932Z{"msg":"[2]R generation[0..12]: [80, 80, 77, 68, 76, 76, 78, 80, 78, 78, 80, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845976426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91252024-02-15T03:01:20.932Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.845991414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91262024-02-15T03:01:20.932Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846005113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91272024-02-15T03:01:20.933Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846019941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91282024-02-15T03:01:20.933Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846033319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91292024-02-15T03:01:20.933Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846047227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91302024-02-15T03:01:20.933Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846061096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91312024-02-15T03:01:20.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846073454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91322024-02-15T03:01:20.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846094321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91332024-02-15T03:01:20.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.84610869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91342024-02-15T03:01:20.933Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846130757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91352024-02-15T03:01:20.933Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846143225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91362024-02-15T03:01:20.933Z{"msg":"74d3ad16-9266-47fd-8595-f27d58a43445 is now active with session: 22289d9d-0e17-4c33-b432-3a8d5c05416c","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846163563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91372024-02-15T03:01:20.933Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846175931Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463,"session_id":"22289d9d-0e17-4c33-b432-3a8d5c05416c"}
91382024-02-15T03:01:20.933Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:19.846195349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463}
91392024-02-15T03:01:20.933ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91402024-02-15T03:01:20.933ZLoading write count information from file "/tmp/test_repair_verify.data"
91412024-02-15T03:01:20.933ZRun Repair workload
91422024-02-15T03:01:20.933Z01/30 Write block 386 len 24576 data: 8 8 7 8 7 7
91432024-02-15T03:01:20.933Z02/30 Write block 302 len 40960 data: 10 11 11 12 9 8 7 8 9 9
91442024-02-15T03:01:20.938Z03/30 Write block 584 len 16384 data: 6 5 6 4
91452024-02-15T03:01:20.938Z04/30 Write block 83 len 12288 data: 5 5 5
91462024-02-15T03:01:20.938Z05/30 Read block 208 len 28672
91472024-02-15T03:01:20.939Z06/30 Write block 557 len 20480 data: 8 8 9 8 6
91482024-02-15T03:01:20.939Z07/30 Write block 283 len 20480 data: 6 8 10 11 10
91492024-02-15T03:01:20.939Z08/30 Flush
91502024-02-15T03:01:20.939Z09/30 Write block 133 len 12288 data: 9 8 8
91512024-02-15T03:01:20.939Z10/30 Read block 486 len 32768
91522024-02-15T03:01:20.939Z11/30 Write block 366 len 36864 data: 8 8 5 4 5 7 6 7 7
91532024-02-15T03:01:20.939Z12/30 Read block 153 len 24576
91542024-02-15T03:01:20.939Z13/30 Read block 296 len 36864
91552024-02-15T03:01:20.945Z14/30 Read block 456 len 8192
91562024-02-15T03:01:20.945Z15/30 Read block 488 len 20480
91572024-02-15T03:01:20.945Z16/30 Write block 185 len 24576 data: 10 10 7 6 9 8
91582024-02-15T03:01:20.945Z17/30 Read block 532 len 36864
91592024-02-15T03:01:20.945Z18/30 Write block 316 len 40960 data: 6 6 6 5 4 4 4 6 6 6
91602024-02-15T03:01:20.945Z19/30 Write block 517 len 16384 data: 5 4 3 3
91612024-02-15T03:01:20.945Z20/30 Flush
91622024-02-15T03:01:20.951Z21/30 Write block 590 len 28672 data: 8 11 9 10 11 9 6
91632024-02-15T03:01:20.951Z22/30 Flush
91642024-02-15T03:01:20.951Z23/30 Write block 209 len 40960 data: 5 5 6 8 8 9 8 7 8 8
91652024-02-15T03:01:20.951Z24/30 Read block 285 len 32768
91662024-02-15T03:01:20.962Z25/30 Write block 479 len 4096 data: 9
91672024-02-15T03:01:20.962Z26/30 Read block 100 len 24576
91682024-02-15T03:01:20.962Z27/30 Read block 12 len 12288
91692024-02-15T03:01:20.962Z28/30 Read block 479 len 40960
91702024-02-15T03:01:20.962Z29/30 Write block 247 len 4096 data: 7
91712024-02-15T03:01:20.962Z30/30 Read block 67 len 28672
91722024-02-15T03:01:20.962Z----------------------------------------------------------------
91732024-02-15T03:01:20.962Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
91742024-02-15T03:01:20.962ZStates: Active Active Active
91752024-02-15T03:01:20.962ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91762024-02-15T03:01:20.962Z 1 Acked 1000 Write 6 Done Done Done false
91772024-02-15T03:01:20.962Z 2 Acked 1001 Write 10 Done Done Done false
91782024-02-15T03:01:20.962Z 3 Acked 1002 Write 4 Done Done Done false
91792024-02-15T03:01:20.962Z 4 Acked 1003 Write 3 Done Done Done false
91802024-02-15T03:01:20.962Z 5 Acked 1004 Read 7 Done Done Done false
91812024-02-15T03:01:20.962Z 6 Acked 1005 Write 5 Done Sent Done false
91822024-02-15T03:01:20.962Z 7 Acked 1006 Write 5 Done Sent Done false
91832024-02-15T03:01:20.962Z 8 Acked 1007 Flush 0 Done Sent Done false
91842024-02-15T03:01:20.962Z 9 Acked 1008 Write 3 Done Sent Done false
91852024-02-15T03:01:20.962Z 10 Acked 1009 Read 8 Done Sent Done false
91862024-02-15T03:01:20.963Z 11 Acked 1010 Write 9 Done Sent Done false
91872024-02-15T03:01:20.963Z 12 Acked 1011 Read 6 Done Sent Done false
91882024-02-15T03:01:20.963Z 13 Acked 1012 Read 9 Done Sent Done false
91892024-02-15T03:01:20.963Z 14 Acked 1013 Read 2 Done Sent Done false
91902024-02-15T03:01:20.963Z 15 Acked 1014 Read 5 Done Sent Done false
91912024-02-15T03:01:20.963Z 16 Acked 1015 Write 6 Done Sent Done false
91922024-02-15T03:01:20.963Z 17 Acked 1016 Read 9 Done Sent Done false
91932024-02-15T03:01:20.963Z 18 Acked 1017 Write 10 Done Sent Done false
91942024-02-15T03:01:20.963Z 19 Acked 1018 Write 4 Done Sent Done false
91952024-02-15T03:01:20.963Z 20 Acked 1019 Flush 0 Done Sent Done false
91962024-02-15T03:01:20.963Z 21 Acked 1020 Write 7 Done Sent Done false
91972024-02-15T03:01:20.963Z 22 Acked 1021 Flush 0 Done Sent Done false
91982024-02-15T03:01:20.963Z 23 Acked 1022 Write 10 Done Sent Done false
91992024-02-15T03:01:20.963Z 24 Acked 1023 Read 8 Done Sent Done false
92002024-02-15T03:01:20.963Z 25 Acked 1024 Write 1 Done Sent Done false
92012024-02-15T03:01:20.963Z 26 Acked 1025 Read 6 Done Sent Done false
92022024-02-15T03:01:20.963Z 27 Acked 1026 Read 3 Done Sent Done false
92032024-02-15T03:01:20.963Z 28 Acked 1027 Read 10 Done Sent Done false
92042024-02-15T03:01:20.963Z 29 Acked 1028 Write 1 Done Sent Done false
92052024-02-15T03:01:20.963Z 30 Acked 1029 Read 7 Done Sent Done false
92062024-02-15T03:01:20.963Z STATES DS:0 DS:1 DS:2 TOTAL
92072024-02-15T03:01:20.963Z New 0 0 0 0
92082024-02-15T03:01:20.963Z Sent 0 25 0 25
92092024-02-15T03:01:20.964Z Done 30 5 30 65
92102024-02-15T03:01:20.964Z Skipped 0 0 0 0
92112024-02-15T03:01:20.964Z Error 0 0 0 0
92122024-02-15T03:01:20.964ZLast Flush: 1021 0 1021
92132024-02-15T03:01:20.964ZDownstairs last five completed:
92142024-02-15T03:01:20.964ZUpstairs last five completed: 30 29 28 27 26
92152024-02-15T03:01:20.964Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:19.879871069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1463}
92162024-02-15T03:01:20.964ZWrote out file "/tmp/test_repair_verify.data"
92172024-02-15T03:01:20.970Z
92182024-02-15T03:01:20.970ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
92192024-02-15T03:01:20.984ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92202024-02-15T03:01:20.984Z 0 000-019 80 80 80 138 138 138 F F F
92212024-02-15T03:01:20.984Z 1 020-039 80 80 80 138 138 138 F F F
92222024-02-15T03:01:20.984Z 2 040-059 77 77 77 135 135 135 F F F
92232024-02-15T03:01:20.985Z 3 060-079 68 68 68 124 124 124 F F F
92242024-02-15T03:01:20.985Z 4 080-099 82 76 82 140 131 140 F T F <---
92252024-02-15T03:01:20.985Z 5 100-119 76 76 76 134 134 134 F F F
92262024-02-15T03:01:21.001Z 6 120-139 82 78 82 141 136 141 F F F <---
92272024-02-15T03:01:21.001Z 7 140-159 80 80 80 138 138 138 F F F
92282024-02-15T03:01:21.001Z 8 160-179 78 78 78 136 136 136 F F F
92292024-02-15T03:01:21.001Z 9 180-199 82 78 82 141 136 141 F F F <---
92302024-02-15T03:01:21.002Z 10 200-219 80 80 80 138 138 138 T F T <---
92312024-02-15T03:01:21.002Z 11 220-239 81 81 81 139 139 139 F F F
92322024-02-15T03:01:21.002Z 12 240-259 78 78 78 136 136 136 T F T <---
92332024-02-15T03:01:21.002Z 13 260-279 74 74 74 130 130 130 F F F
92342024-02-15T03:01:21.002Z 14 280-299 82 76 82 140 131 140 F F F <---
92352024-02-15T03:01:21.002Z 15 300-319 82 68 82 141 124 141 F T F <---
92362024-02-15T03:01:21.002Z 16 320-339 82 73 82 141 129 141 F F F <---
92372024-02-15T03:01:21.002Z 17 340-359 78 78 78 136 136 136 F F F
92382024-02-15T03:01:21.002Z 18 360-379 82 78 82 141 136 141 F F F <---
92392024-02-15T03:01:21.002Z 19 380-399 82 77 82 140 135 140 F T F <---
92402024-02-15T03:01:21.002Z 20 400-419 80 80 80 137 137 137 F F F
92412024-02-15T03:01:21.002Z 21 420-439 76 76 76 133 133 133 F F F
92422024-02-15T03:01:21.002Z 22 440-459 81 81 81 139 139 139 F F F
92432024-02-15T03:01:21.002Z 23 460-479 80 80 80 138 138 138 T F T <---
92442024-02-15T03:01:21.002Z 24 480-499 81 81 81 139 139 139 F F F
92452024-02-15T03:01:21.002Z 25 500-519 82 80 82 141 138 141 F F F <---
92462024-02-15T03:01:21.002Z 26 520-539 82 80 82 141 138 141 F F F <---
92472024-02-15T03:01:21.002Z 27 540-559 82 78 82 140 136 140 F F F <---
92482024-02-15T03:01:21.002Z 28 560-579 82 76 82 140 134 140 F F F <---
92492024-02-15T03:01:21.003Z 29 580-599 82 78 82 142 136 142 F T F <---
92502024-02-15T03:01:21.003ZMax gen: 82, Max flush: 142
92512024-02-15T03:01:21.003ZError: Difference in extent metadata found!
92522024-02-15T03:01:21.003ZA Difference in extent metadata is expected here
92532024-02-15T03:01:21.003ZOn loop 41
92542024-02-15T03:01:21.003Z
92552024-02-15T03:01:21.003ZVerifying data now
92562024-02-15T03:01:26.772ZNo change after verify
92572024-02-15T03:01:26.775ZLoop: 41 Downstairs dump after verify (and repair):
92582024-02-15T03:01:26.787ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92592024-02-15T03:01:26.787Z 0 000-019 80 80 80 138 138 138 F F F
92602024-02-15T03:01:26.787Z 1 020-039 80 80 80 138 138 138 F F F
92612024-02-15T03:01:26.787Z 2 040-059 77 77 77 135 135 135 F F F
92622024-02-15T03:01:26.788Z 3 060-079 68 68 68 124 124 124 F F F
92632024-02-15T03:01:26.793Z 4 080-099 82 82 82 140 140 140 F F F
92642024-02-15T03:01:26.793Z 5 100-119 76 76 76 134 134 134 F F F
92652024-02-15T03:01:26.793Z 6 120-139 82 82 82 141 141 141 F F F
92662024-02-15T03:01:26.793Z 7 140-159 80 80 80 138 138 138 F F F
92672024-02-15T03:01:26.793Z 8 160-179 78 78 78 136 136 136 F F F
92682024-02-15T03:01:26.793Z 9 180-199 82 82 82 141 141 141 F F F
92692024-02-15T03:01:26.793Z 10 200-219 83 83 83 143 143 143 F F F
92702024-02-15T03:01:26.793Z 11 220-239 81 81 81 139 139 139 F F F
92712024-02-15T03:01:26.793Z 12 240-259 83 83 83 143 143 143 F F F
92722024-02-15T03:01:26.793Z 13 260-279 74 74 74 130 130 130 F F F
92732024-02-15T03:01:26.794Z 14 280-299 82 82 82 140 140 140 F F F
92742024-02-15T03:01:26.794Z 15 300-319 82 82 82 141 141 141 F F F
92752024-02-15T03:01:26.794Z 16 320-339 82 82 82 141 141 141 F F F
92762024-02-15T03:01:26.794Z 17 340-359 78 78 78 136 136 136 F F F
92772024-02-15T03:01:26.794Z 18 360-379 82 82 82 141 141 141 F F F
92782024-02-15T03:01:26.794Z 19 380-399 82 82 82 140 140 140 F F F
92792024-02-15T03:01:26.794Z 20 400-419 80 80 80 137 137 137 F F F
92802024-02-15T03:01:26.794Z 21 420-439 76 76 76 133 133 133 F F F
92812024-02-15T03:01:26.794Z 22 440-459 81 81 81 139 139 139 F F F
92822024-02-15T03:01:26.794Z 23 460-479 83 83 83 143 143 143 F F F
92832024-02-15T03:01:26.794Z 24 480-499 81 81 81 139 139 139 F F F
92842024-02-15T03:01:26.794Z 25 500-519 82 82 82 141 141 141 F F F
92852024-02-15T03:01:26.794Z 26 520-539 82 82 82 141 141 141 F F F
92862024-02-15T03:01:26.794Z 27 540-559 82 82 82 140 140 140 F F F
92872024-02-15T03:01:26.794Z 28 560-579 82 82 82 140 140 140 F F F
92882024-02-15T03:01:26.794Z 29 580-599 82 82 82 142 142 142 F F F
92892024-02-15T03:01:26.794ZMax gen: 83, Max flush: 143
92902024-02-15T03:01:26.794Z
92912024-02-15T03:01:26.794ZBegin loop 42
92922024-02-15T03:01:26.795ZDownstairs to restart: 1
92932024-02-15T03:01:26.814Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.734773806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471}
92942024-02-15T03:01:26.814Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735068229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471}
92952024-02-15T03:01:26.814Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735095966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471}
92962024-02-15T03:01:26.814Z{Crucible runtime is spawned
92972024-02-15T03:01:26.821Z"msg":"Crucible 028c91ba-39da-4ccf-970d-fa77546679f7 has session id: b5f90388-195d-4c10-9ef8-0b5405155d91","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735137591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
92982024-02-15T03:01:26.821Z{"msg":"Upstairs opts: Upstairs UUID: 028c91ba-39da-4ccf-970d-fa77546679f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735188654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
92992024-02-15T03:01:26.821Z{"msg":"Crucible stats registered with UUID: 028c91ba-39da-4ccf-970d-fa77546679f7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735216331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93002024-02-15T03:01:26.821Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735248177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471}
93012024-02-15T03:01:26.822Z{"msg":"028c91ba-39da-4ccf-970d-fa77546679f7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735388769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93022024-02-15T03:01:26.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735419085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93032024-02-15T03:01:26.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735444882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93042024-02-15T03:01:26.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735484427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93052024-02-15T03:01:26.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735510734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93062024-02-15T03:01:26.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735555259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93072024-02-15T03:01:26.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735584045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93082024-02-15T03:01:26.822Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735617681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93092024-02-15T03:01:26.822Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735645947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93102024-02-15T03:01:26.822Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735673224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93112024-02-15T03:01:26.822Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735756963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93122024-02-15T03:01:26.822Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.735834904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93132024-02-15T03:01:26.822Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:25.735897836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93142024-02-15T03:01:26.822Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:25.735949459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93152024-02-15T03:01:26.822Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:25.736079163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93162024-02-15T03:01:26.822Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.736109179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93172024-02-15T03:01:26.822Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.736124697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93182024-02-15T03:01:26.822Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.736249332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93192024-02-15T03:01:26.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.736827039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93202024-02-15T03:01:26.822Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.736878393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93212024-02-15T03:01:26.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.736908969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93222024-02-15T03:01:26.822Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.736938935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93232024-02-15T03:01:26.822Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.737501715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93242024-02-15T03:01:26.822Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.737663074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93252024-02-15T03:01:26.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.738047096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93262024-02-15T03:01:26.823Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.73809951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93272024-02-15T03:01:26.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.738141035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93282024-02-15T03:01:26.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.738164552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93292024-02-15T03:01:26.823Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.738188838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93302024-02-15T03:01:26.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:25.738210406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93312024-02-15T03:01:27.818Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.73691974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93322024-02-15T03:01:27.826Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.737064872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93332024-02-15T03:01:27.826Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.737949591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93342024-02-15T03:01:27.826Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.737989286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93352024-02-15T03:01:27.826Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.738485044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93362024-02-15T03:01:27.826Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739085919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93372024-02-15T03:01:27.826Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739143022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93382024-02-15T03:01:27.826Z{"msg":"[0]R flush_numbers[0..12]: [138, 138, 135, 124, 140, 134, 141, 138, 136, 141, 143, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739173338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93392024-02-15T03:01:27.826Z{"msg":"[0]R generation[0..12]: [80, 80, 77, 68, 82, 76, 82, 80, 78, 82, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739203344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93402024-02-15T03:01:27.826Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739222732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93412024-02-15T03:01:27.826Z{"msg":"[1]R flush_numbers[0..12]: [138, 138, 135, 124, 140, 134, 141, 138, 136, 141, 143, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739250348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93422024-02-15T03:01:27.826Z{"msg":"[1]R generation[0..12]: [80, 80, 77, 68, 82, 76, 82, 80, 78, 82, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739268336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93432024-02-15T03:01:27.826Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739295713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93442024-02-15T03:01:27.826Z{"msg":"[2]R flush_numbers[0..12]: [138, 138, 135, 124, 140, 134, 141, 138, 136, 141, 143, 139]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
93452024-02-15T03:01:27.826Zlevel":30,"time":"2024-02-15T03:01:26.73931868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93462024-02-15T03:01:27.826Z{"msg":"[2]R generation[0..12]: [80, 80, 77, 68, 82, 76, 82, 80, 78, 82, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739344907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93472024-02-15T03:01:27.827Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739363104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93482024-02-15T03:01:27.827Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739379712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93492024-02-15T03:01:27.827Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.73939762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93502024-02-15T03:01:27.827Z{"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739422707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93512024-02-15T03:01:27.827Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739439685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93522024-02-15T03:01:27.827Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739456863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93532024-02-15T03:01:27.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739472511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93542024-02-15T03:01:27.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739490078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93552024-02-15T03:01:27.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739515065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93562024-02-15T03:01:27.827Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739532613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93572024-02-15T03:01:27.827Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739548811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93582024-02-15T03:01:27.827Z{"msg":"028c91ba-39da-4ccf-970d-fa77546679f7 is now active with session: b5f90388-195d-4c10-9ef8-0b5405155d91","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739572958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93592024-02-15T03:01:27.827Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739588116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471,"session_id":"b5f90388-195d-4c10-9ef8-0b5405155d91"}
93602024-02-15T03:01:27.827Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:26.739611873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471}
93612024-02-15T03:01:27.827ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93622024-02-15T03:01:27.827ZLoading write count information from file "/tmp/test_repair_verify.data"
93632024-02-15T03:01:27.827ZRun Repair workload
93642024-02-15T03:01:27.827Z01/30 Write block 100 len 24576 data: 8 9 9 6 7 7
93652024-02-15T03:01:27.827Z02/30 Read block 114 len 36864
93662024-02-15T03:01:27.827Z03/30 Flush
93672024-02-15T03:01:27.827Z04/30 Write block 388 len 16384 data: 8 9 8 8
93682024-02-15T03:01:27.827Z05/30 Write block 162 len 20480 data: 10 7 5 7 8
93692024-02-15T03:01:27.831Z06/30 Read block 365 len 16384
93702024-02-15T03:01:27.831Z07/30 Read block 352 len 20480
93712024-02-15T03:01:27.831Z08/30 Write block 564 len 8192 data: 7 7
93722024-02-15T03:01:27.831Z09/30 Read block 149 len 32768
93732024-02-15T03:01:27.831Z10/30 Write block 506 len 24576 data: 7 5 6 8 7 8
93742024-02-15T03:01:27.831Z11/30 Write block 57 len 32768 data: 7 6 4 5 5 6 7 7
93752024-02-15T03:01:27.831Z12/30 Read block 467 len 32768
93762024-02-15T03:01:27.831Z13/30 Write block 409 len 28672 data: 14 14 13 12 12 10 8
93772024-02-15T03:01:27.834Z14/30 Read block 237 len 40960
93782024-02-15T03:01:27.834Z15/30 Write block 337 len 24576 data: 10 10 7 5 6 6
93792024-02-15T03:01:27.835Z16/30 Write block 55 len 16384 data: 8 7 8 7
93802024-02-15T03:01:27.835Z17/30 Read block 393 len 32768
93812024-02-15T03:01:27.838Z18/30 Write block 283 len 40960 data: 7 9 11 12 11 8 8 12 10 9
93822024-02-15T03:01:27.838Z19/30 Flush
93832024-02-15T03:01:27.841Z20/30 Write block 367 len 36864 data: 9 6 5 6 8 7 8 8 5
93842024-02-15T03:01:27.844Z21/30 Write block 431 len 20480 data: 6 5 4 4 6
93852024-02-15T03:01:27.844Z22/30 Read block 425 len 16384
93862024-02-15T03:01:27.844Z23/30 Flush
93872024-02-15T03:01:27.847Z24/30 Flush
93882024-02-15T03:01:27.847Z25/30 Write block 282 len 4096 data: 3
93892024-02-15T03:01:27.847Z26/30 Read block 57 len 16384
93902024-02-15T03:01:27.847Z27/30 Read block 234 len 28672
93912024-02-15T03:01:27.851Z28/30 Read block 142 len 32768
93922024-02-15T03:01:27.851Z29/30 Write block 289 len 28672 data: 9 13 11 10 10 10 7
93932024-02-15T03:01:27.851Z30/30 Read block 369 len 36864
93942024-02-15T03:01:27.854Z----------------------------------------------------------------
93952024-02-15T03:01:27.854Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:27
93962024-02-15T03:01:27.854ZStates: Active Active Active
93972024-02-15T03:01:27.854ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93982024-02-15T03:01:27.854Z 4 Acked 1003 Write 4 Done Done Done false
93992024-02-15T03:01:27.854Z 5 Acked 1004 Write 5 Done Sent Done false
94002024-02-15T03:01:27.854Z 6 Acked 1005 Read 4 Done Sent Done false
94012024-02-15T03:01:27.854Z 7 Acked 1006 Read 5 Done Sent Done false
94022024-02-15T03:01:27.854Z 8 Acked 1007 Write 2 Done Sent Done false
94032024-02-15T03:01:27.854Z 9 Acked 1008 Read 8 Done Sent Done false
94042024-02-15T03:01:27.854Z 10 Acked 1009 Write 6 Done Sent Done false
94052024-02-15T03:01:27.854Z 11 Acked 1010 Write 8 Done Sent Done false
94062024-02-15T03:01:27.855Z 12 Acked 1011 Read 8 Done Sent Done false
94072024-02-15T03:01:27.855Z 13 Acked 1012 Write 7 Done Sent Done false
94082024-02-15T03:01:27.855Z 14 Acked 1013 Read 10 Done Sent Done false
94092024-02-15T03:01:27.855Z 15 Acked 1014 Write 6 Done Sent Done false
94102024-02-15T03:01:27.855Z 16 Acked 1015 Write 4 Done Sent Done false
94112024-02-15T03:01:27.855Z 17 Acked 1016 Read 8 Done Sent Done false
94122024-02-15T03:01:27.855Z 18 Acked 1017 Write 10 Done Sent Done false
94132024-02-15T03:01:27.855Z 19 Acked 1018 Flush 0 Done Sent Done false
94142024-02-15T03:01:27.855Z 20 Acked 1019 Write 9 Done Sent Done false
94152024-02-15T03:01:27.855Z 21 Acked 1020 Write 5 Done Sent Done false
94162024-02-15T03:01:27.855Z 22 Acked 1021 Read 4 Done Sent Done false
94172024-02-15T03:01:27.855Z 23 Acked 1022 Flush 0 Done Sent Done false
94182024-02-15T03:01:27.855Z 24 Acked 1023 Flush 0 Done Sent Done false
94192024-02-15T03:01:27.855Z 25 Acked 1024 Write 1 Done Sent Done false
94202024-02-15T03:01:27.855Z 26 Acked 1025 Read 4 Done Sent Done false
94212024-02-15T03:01:27.855Z 27 Acked 1026 Read 7 Done Sent Done false
94222024-02-15T03:01:27.855Z 28 Acked 1027 Read 8 Done Sent Done false
94232024-02-15T03:01:27.855Z 29 Acked 1028 Write 7 Done Sent Done false
94242024-02-15T03:01:27.855Z 30 Acked 1029 Read 9 Done Sent Sent false
94252024-02-15T03:01:27.855Z STATES DS:0 DS:1 DS:2 TOTAL
94262024-02-15T03:01:27.855Z New 0 0 0 0
94272024-02-15T03:01:27.855Z Sent 0 26 1 27
94282024-02-15T03:01:27.855Z Done 27 1 26 54
94292024-02-15T03:01:27.855Z Skipped 0 0 0 0
94302024-02-15T03:01:27.855Z Error 0 0 0 0
94312024-02-15T03:01:27.856ZLast Flush: 1023 1002 1023
94322024-02-15T03:01:27.856ZDownstairs last five completed: 1002 1001 1000
94332024-02-15T03:01:27.856ZUpstairs last five completed: 30 29 28 27 26
94342024-02-15T03:01:27.856Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:26.774423561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1471}
94352024-02-15T03:01:27.856ZWrote out file "/tmp/test_repair_verify.data"
94362024-02-15T03:01:27.859Z
94372024-02-15T03:01:27.864ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94382024-02-15T03:01:27.880ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94392024-02-15T03:01:27.880Z 0 000-019 80 80 80 138 138 138 F F F
94402024-02-15T03:01:27.880Z 1 020-039 80 80 80 138 138 138 F F F
94412024-02-15T03:01:27.880Z 2 040-059 84 77 84 145 135 145 F F F <---
94422024-02-15T03:01:27.880Z 3 060-079 84 68 84 145 124 145 F F F <---
94432024-02-15T03:01:27.880Z 4 080-099 82 82 82 140 140 140 F F F
94442024-02-15T03:01:27.880Z 5 100-119 84 84 84 144 144 144 F F F
94452024-02-15T03:01:27.880Z 6 120-139 82 82 82 141 141 141 F F F
94462024-02-15T03:01:27.880Z 7 140-159 80 80 80 138 138 138 F F F
94472024-02-15T03:01:27.880Z 8 160-179 84 78 84 145 136 145 F F F <---
94482024-02-15T03:01:27.880Z 9 180-199 82 82 82 141 141 141 F F F
94492024-02-15T03:01:27.881Z 10 200-219 83 83 83 143 143 143 F F F
94502024-02-15T03:01:27.881Z 11 220-239 81 81 81 139 139 139 F F F
94512024-02-15T03:01:27.881Z 12 240-259 83 83 83 143 143 143 F F F
94522024-02-15T03:01:27.894Z 13 260-279 74 74 74 130 130 130 F F F
94532024-02-15T03:01:27.894Z 14 280-299 84 82 84 145 140 145 T F T <---
94542024-02-15T03:01:27.894Z 15 300-319 82 82 82 141 141 141 F F F
94552024-02-15T03:01:27.894Z 16 320-339 84 82 84 145 141 145 F F F <---
94562024-02-15T03:01:27.894Z 17 340-359 84 78 84 145 136 145 F F F <---
94572024-02-15T03:01:27.894Z 18 360-379 84 82 84 146 141 146 F F F <---
94582024-02-15T03:01:27.894Z 19 380-399 84 82 84 145 140 145 F T F <---
94592024-02-15T03:01:27.894Z 20 400-419 84 80 84 145 137 145 F F F <---
94602024-02-15T03:01:27.894Z 21 420-439 84 76 84 146 133 146 F F F <---
94612024-02-15T03:01:27.894Z 22 440-459 81 81 81 139 139 139 F F F
94622024-02-15T03:01:27.894Z 23 460-479 83 83 83 143 143 143 F F F
94632024-02-15T03:01:27.894Z 24 480-499 81 81 81 139 139 139 F F F
94642024-02-15T03:01:27.894Z 25 500-519 84 82 84 145 141 145 F F F <---
94652024-02-15T03:01:27.895Z 26 520-539 82 82 82 141 141 141 F F F
94662024-02-15T03:01:27.895Z 27 540-559 82 82 82 140 140 140 F F F
94672024-02-15T03:01:27.895Z 28 560-579 84 82 84 145 140 145 F F F <---
94682024-02-15T03:01:27.895Z 29 580-599 82 82 82 142 142 142 F F F
94692024-02-15T03:01:27.895ZMax gen: 84, Max flush: 146
94702024-02-15T03:01:27.895ZError: Difference in extent metadata found!
94712024-02-15T03:01:27.895ZA Difference in extent metadata is expected here
94722024-02-15T03:01:27.896ZOn loop 42
94732024-02-15T03:01:27.896Z
94742024-02-15T03:01:27.896ZVerifying data now
94752024-02-15T03:01:33.906ZNo change after verify
94762024-02-15T03:01:33.906ZLoop: 42 Downstairs dump after verify (and repair):
94772024-02-15T03:01:33.919ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94782024-02-15T03:01:33.920Z 0 000-019 80 80 80 138 138 138 F F F
94792024-02-15T03:01:33.920Z 1 020-039 80 80 80 138 138 138 F F F
94802024-02-15T03:01:33.920Z 2 040-059 84 84 84 145 145 145 F F F
94812024-02-15T03:01:33.920Z 3 060-079 84 84 84 145 145 145 F F F
94822024-02-15T03:01:33.920Z 4 080-099 82 82 82 140 140 140 F F F
94832024-02-15T03:01:33.920Z 5 100-119 84 84 84 144 144 144 F F F
94842024-02-15T03:01:33.921Z 6 120-139 82 82 82 141 141 141 F F F
94852024-02-15T03:01:33.921Z 7 140-159 80 80 80 138 138 138 F F F
94862024-02-15T03:01:33.921Z 8 160-179 84 84 84 145 145 145 F F F
94872024-02-15T03:01:33.921Z 9 180-199 82 82 82 141 141 141 F F F
94882024-02-15T03:01:33.921Z 10 200-219 83 83 83 143 143 143 F F F
94892024-02-15T03:01:33.921Z 11 220-239 81 81 81 139 139 139 F F F
94902024-02-15T03:01:33.922Z 12 240-259 83 83 83 143 143 143 F F F
94912024-02-15T03:01:33.922Z 13 260-279 74 74 74 130 130 130 F F F
94922024-02-15T03:01:33.922Z 14 280-299 85 85 85 147 147 147 F F F
94932024-02-15T03:01:33.922Z 15 300-319 82 82 82 141 141 141 F F F
94942024-02-15T03:01:33.922Z 16 320-339 84 84 84 145 145 145 F F F
94952024-02-15T03:01:33.922Z 17 340-359 84 84 84 145 145 145 F F F
94962024-02-15T03:01:33.923Z 18 360-379 84 84 84 146 146 146 F F F
94972024-02-15T03:01:33.923Z 19 380-399 84 84 84 145 145 145 F F F
94982024-02-15T03:01:33.923Z 20 400-419 84 84 84 145 145 145 F F F
94992024-02-15T03:01:33.923Z 21 420-439 84 84 84 146 146 146 F F F
95002024-02-15T03:01:33.923Z 22 440-459 81 81 81 139 139 139 F F F
95012024-02-15T03:01:33.923Z 23 460-479 83 83 83 143 143 143 F F F
95022024-02-15T03:01:33.923Z 24 480-499 81 81 81 139 139 139 F F F
95032024-02-15T03:01:33.923Z 25 500-519 84 84 84 145 145 145 F F F
95042024-02-15T03:01:33.924Z 26 520-539 82 82 82 141 141 141 F F F
95052024-02-15T03:01:33.924Z 27 540-559 82 82 82 140 140 140 F F F
95062024-02-15T03:01:33.924Z 28 560-579 84 84 84 145 145 145 F F F
95072024-02-15T03:01:33.925Z 29 580-599 82 82 82 142 142 142 F F F
95082024-02-15T03:01:33.925ZMax gen: 85, Max flush: 147
95092024-02-15T03:01:33.926Z
95102024-02-15T03:01:33.926ZBegin loop 43
95112024-02-15T03:01:33.926ZDownstairs to restart: 0
95122024-02-15T03:01:33.926Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459085021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479}
95132024-02-15T03:01:33.926Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.45933238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479}
95142024-02-15T03:01:33.927Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459363146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479}
95152024-02-15T03:01:33.927Z{"msg":"Crucible 0ecf3896-e899-45c2-be5f-9f93c8bb4ed3 has session id: 8bd67c53-bca6-487e-b370-5325b5dd6242","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
95162024-02-15T03:01:33.927Ztime":"2024-02-15T03:01:32.459398722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95172024-02-15T03:01:33.928Z{"msg":"Upstairs opts: Upstairs UUID: 0ecf3896-e899-45c2-be5f-9f93c8bb4ed3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459443886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95182024-02-15T03:01:33.928Z{"msg":"Crucible stats registered with UUID: 0ecf3896-e899-45c2-be5f-9f93c8bb4ed3","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459473632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95192024-02-15T03:01:33.928Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.45949491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479}
95202024-02-15T03:01:33.928Z{"msg":"0ecf3896-e899-45c2-be5f-9f93c8bb4ed3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459625743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95212024-02-15T03:01:33.928Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459666558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95222024-02-15T03:01:33.928Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459694125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95232024-02-15T03:01:33.929Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459734999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95242024-02-15T03:01:33.929Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459763956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95252024-02-15T03:01:33.929Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459825118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95262024-02-15T03:01:33.930Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459862423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95272024-02-15T03:01:33.930Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.45988777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95282024-02-15T03:01:33.930Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459922926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95292024-02-15T03:01:33.930Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459950093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95302024-02-15T03:01:33.930Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.459989977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95312024-02-15T03:01:33.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.460063648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95322024-02-15T03:01:33.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.460093664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95332024-02-15T03:01:33.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.460780158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95342024-02-15T03:01:33.931Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.460817214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95352024-02-15T03:01:33.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.460833572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95362024-02-15T03:01:33.931Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.460858708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95372024-02-15T03:01:33.932Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.461385682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95382024-02-15T03:01:33.932Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.461546682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95392024-02-15T03:01:33.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.461894509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95402024-02-15T03:01:33.932Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.461940893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95412024-02-15T03:01:33.932Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.461991717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95422024-02-15T03:01:33.932Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.462037731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95432024-02-15T03:01:33.932Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.462075596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95442024-02-15T03:01:33.932Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.462117321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95452024-02-15T03:01:33.933Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.462156506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95462024-02-15T03:01:33.933Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.462193511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95472024-02-15T03:01:33.933Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.462709257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95482024-02-15T03:01:33.933Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.46316422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95492024-02-15T03:01:33.933Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463195476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95502024-02-15T03:01:33.933Z{"msg":"[0]R flush_numbers[0..12]: [138, 138, 145, 145, 140, 144, 141, 138, 145, 141, 143, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463221452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95512024-02-15T03:01:33.934Z{"msg":"[0]R generation[0..12]: [80, 80, 84, 84, 82, 84, 82, 80, 84, 82, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463252309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95522024-02-15T03:01:33.934Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463278085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95532024-02-15T03:01:33.934Z{"msg":"[1]R flush_numbers[0..12]: [138, 138, 145, 145, 140, 144, 141, 138, 145, 141, 143, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463306842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95542024-02-15T03:01:33.934Z{"msg":"[1]R generation[0..12]: [80, 80, 84, 84, 82, 84, 82, 80, 84, 82, 83, 81]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
95552024-02-15T03:01:33.934Z2024-02-15T03:01:32.463331389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95562024-02-15T03:01:33.935Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463369064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95572024-02-15T03:01:33.935Z{"msg":"[2]R flush_numbers[0..12]: [138, 138, 145, 145, 140, 144, 141, 138, 145, 141, 143, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.46340185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95582024-02-15T03:01:33.935Z{"msg":"[2]R generation[0..12]: [80, 80, 84, 84, 82, 84, 82, 80, 84, 82, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463427487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95592024-02-15T03:01:33.935Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463453194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95602024-02-15T03:01:33.935Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.46348215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95612024-02-15T03:01:33.935Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463515176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95622024-02-15T03:01:33.935Z{"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463539513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95632024-02-15T03:01:33.935Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.4635636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95642024-02-15T03:01:33.936Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463593486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95652024-02-15T03:01:33.936Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463615563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95662024-02-15T03:01:33.936Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.46364393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95672024-02-15T03:01:33.936Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463667807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95682024-02-15T03:01:33.936Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463694733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95692024-02-15T03:01:33.936Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95702024-02-15T03:01:33.937Z,"time":"2024-02-15T03:01:32.46371713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95712024-02-15T03:01:33.937Z{"msg":"0ecf3896-e899-45c2-be5f-9f93c8bb4ed3 is now active with session: 8bd67c53-bca6-487e-b370-5325b5dd6242","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463753636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95722024-02-15T03:01:33.937Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463783902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479,"session_id":"8bd67c53-bca6-487e-b370-5325b5dd6242"}
95732024-02-15T03:01:33.937Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:32.463807209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479}
95742024-02-15T03:01:33.937ZLoading write count information from file "/tmp/test_repair_verify.data"
95752024-02-15T03:01:33.937ZRun Repair workload
95762024-02-15T03:01:33.937Z01/30 Write block 375 len 20480 data: 6 6 6 7 6
95772024-02-15T03:01:33.956Z02/30 Write block 411 len 20480 data: 14 13 13 11 9
95782024-02-15T03:01:33.957Z03/30 Read block 185 len 12288
95792024-02-15T03:01:33.957Z04/30 Read block 220 len 28672
95802024-02-15T03:01:33.957Z05/30 Write block 231 len 20480 data: 7 8 8 9 8
95812024-02-15T03:01:33.957Z06/30 Read block 492 len 4096
95822024-02-15T03:01:33.958Z07/30 Write block 57 len 24576 data: 9 8 5 6 6 7
95832024-02-15T03:01:33.958Z08/30 Read block 402 len 4096
95842024-02-15T03:01:33.958Z09/30 Read block 189 len 12288
95852024-02-15T03:01:33.958Z10/30 Read block 120 len 32768
95862024-02-15T03:01:33.958Z11/30 Read block 232 len 28672
95872024-02-15T03:01:33.958Z12/30 Write block 478 len 16384 data: 9 10 8 8
95882024-02-15T03:01:33.958Z13/30 Read block 150 len 4096
95892024-02-15T03:01:33.958Z14/30 Read block 95 len 20480
95902024-02-15T03:01:33.959Z15/30 Write block 290 len 24576 data: 14 12 11 11 11 8
95912024-02-15T03:01:33.959Z16/30 Read block 238 len 8192
95922024-02-15T03:01:33.959Z17/30 Read block 58 len 28672
95932024-02-15T03:01:33.959Z18/30 Read block 251 len 40960
95942024-02-15T03:01:33.959Z19/30 Flush
95952024-02-15T03:01:33.959Z20/30 Write block 14 len 24576 data: 5 3 3 4 6 5
95962024-02-15T03:01:33.959Z21/30 Read block 44 len 28672
95972024-02-15T03:01:33.959Z22/30 Write block 26 len 16384 data: 4 5 6 5
95982024-02-15T03:01:33.959Z23/30 Flush
95992024-02-15T03:01:33.960Z24/30 Write block 410 len 16384 data: 15 15 14 14
96002024-02-15T03:01:33.960Z25/30 Flush
96012024-02-15T03:01:33.960Z26/30 Write block 42 len 20480 data: 6 5 8 8 8
96022024-02-15T03:01:33.960Z27/30 Read block 84 len 20480
96032024-02-15T03:01:33.960Z28/30 Write block 325 len 24576 data: 7 8 7 8 8 8
96042024-02-15T03:01:33.961Z29/30 Read block 573 len 8192
96052024-02-15T03:01:33.961Z30/30 Read block 320 len 24576
96062024-02-15T03:01:33.961Z----------------------------------------------------------------
96072024-02-15T03:01:33.961Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
96082024-02-15T03:01:33.961ZStates: Active Active Active
96092024-02-15T03:01:33.961ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
96102024-02-15T03:01:33.961Z 1 Acked 1000 Write 5 Done Done Done false
96112024-02-15T03:01:33.961Z 2 Acked 1001 Write 5 Done Done Done false
96122024-02-15T03:01:33.962Z 3 Acked 1002 Read 3 Done Done Done false
96132024-02-15T03:01:33.962Z 4 Acked 1003 Read 7 Sent Done Done false
96142024-02-15T03:01:33.962Z 5 Acked 1004 Write 5 Sent Done Done false
96152024-02-15T03:01:33.962Z 6 Acked 1005 Read 1 Sent Done Done false
96162024-02-15T03:01:33.962Z 7 Acked 1006 Write 6 Sent Done Done false
96172024-02-15T03:01:33.962Z 8 Acked 1007 Read 1 Sent Done Done false
96182024-02-15T03:01:33.962Z 9 Acked 1008 Read 3 Sent Done Done false
96192024-02-15T03:01:33.962Z 10 Acked 1009 Read 8 Sent Done Done false
96202024-02-15T03:01:33.963Z 11 Acked 1010 Read 7 Sent Done Done false
96212024-02-15T03:01:33.963Z 12 Acked 1011 Write 4 Sent Done Done false
96222024-02-15T03:01:33.963Z 13 Acked 1012 Read 1 Sent Done Done false
96232024-02-15T03:01:33.963Z 14 Acked 1013 Read 5 Sent Done Done false
96242024-02-15T03:01:33.963Z 15 Acked 1014 Write 6 Sent Done Done false
96252024-02-15T03:01:33.963Z 16 Acked 1015 Read 2 Sent Done Done false
96262024-02-15T03:01:33.964Z 17 Acked 1016 Read 7 Sent Done Done false
96272024-02-15T03:01:33.964Z 18 Acked 1017 Read 10 Sent Done Done false
96282024-02-15T03:01:33.964Z 19 Acked 1018 Flush 0 Sent Done Done false
96292024-02-15T03:01:33.964Z 20 Acked 1019 Write 6 Sent Done Done false
96302024-02-15T03:01:33.964Z 21 Acked 1020 Read 7 Sent Done Done false
96312024-02-15T03:01:33.964Z 22 Acked 1021 Write 4 Sent Done Done false
96322024-02-15T03:01:33.964Z 23 Acked 1022 Flush 0 Sent Done Done false
96332024-02-15T03:01:33.964Z 24 Acked 1023 Write 4 Sent Done Done false
96342024-02-15T03:01:33.964Z 25 Acked 1024 Flush 0 Sent Done Done false
96352024-02-15T03:01:33.965Z 26 Acked 1025 Write 5 Sent Done Done false
96362024-02-15T03:01:33.965Z 27 Acked 1026 Read 5 Sent Done Done false
96372024-02-15T03:01:33.965Z 28 Acked 1027 Write 6 Sent Done Done false
96382024-02-15T03:01:33.965Z 29 Acked 1028 Read 2 Sent Done Done false
96392024-02-15T03:01:33.965Z 30 Acked 1029 Read 6 Sent Sent Done false
96402024-02-15T03:01:33.965Z STATES DS:0 DS:1 DS:2 TOTAL
96412024-02-15T03:01:33.965Z New 0 0 0 0
96422024-02-15T03:01:33.966Z Sent 27 1 0 28
96432024-02-15T03:01:33.966Z Done 3 29 30 62
96442024-02-15T03:01:33.966Z Skipped 0 0 0 0
96452024-02-15T03:01:33.966Z Error 0 0 0 0
96462024-02-15T03:01:33.966ZLast Flush: 0 1024 1024
96472024-02-15T03:01:33.966ZDownstairs last five completed:
96482024-02-15T03:01:33.967ZUpstairs last five completed: 30 29 28 27 26
96492024-02-15T03:01:33.967Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:32.495400301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1479}
96502024-02-15T03:01:33.967ZWrote out file "/tmp/test_repair_verify.data"
96512024-02-15T03:01:33.967Z
96522024-02-15T03:01:33.967ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96532024-02-15T03:01:33.968ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96542024-02-15T03:01:33.968Z 0 000-019 80 86 86 138 149 149 F F F <---
96552024-02-15T03:01:33.968Z 1 020-039 80 86 86 138 149 149 F F F <---
96562024-02-15T03:01:33.968Z 2 040-059 84 86 86 145 148 148 F T T <---
96572024-02-15T03:01:33.968Z 3 060-079 84 86 86 145 148 148 F F F <---
96582024-02-15T03:01:33.968Z 4 080-099 82 82 82 140 140 140 F F F
96592024-02-15T03:01:33.968Z 5 100-119 84 84 84 144 144 144 F F F
96602024-02-15T03:01:33.968Z 6 120-139 82 82 82 141 141 141 F F F
96612024-02-15T03:01:33.969Z 7 140-159 80 80 80 138 138 138 F F F
96622024-02-15T03:01:33.969Z 8 160-179 84 84 84 145 145 145 F F F
96632024-02-15T03:01:33.969Z 9 180-199 82 82 82 141 141 141 F F F
96642024-02-15T03:01:33.969Z 10 200-219 83 83 83 143 143 143 F F F
96652024-02-15T03:01:33.969Z 11 220-239 81 86 86 139 148 148 F F F <---
96662024-02-15T03:01:33.969Z 12 240-259 83 83 83 143 143 143 F F F
96672024-02-15T03:01:33.969Z 13 260-279 74 74 74 130 130 130 F F F
96682024-02-15T03:01:33.969Z 14 280-299 85 86 86 147 148 148 F F F <---
96692024-02-15T03:01:33.969Z 15 300-319 82 82 82 141 141 141 F F F
96702024-02-15T03:01:33.969Z 16 320-339 84 84 84 145 145 145 F T T <---
96712024-02-15T03:01:33.970Z 17 340-359 84 84 84 145 145 145 F F F
96722024-02-15T03:01:33.970Z 18 360-379 84 86 86 146 148 148 T F F <---
96732024-02-15T03:01:33.970Z 19 380-399 84 84 84 145 145 145 F F F
96742024-02-15T03:01:33.970Z 20 400-419 84 86 86 145 150 150 T F F <---
96752024-02-15T03:01:33.970Z 21 420-439 84 84 84 146 146 146 F F F
96762024-02-15T03:01:33.971Z 22 440-459 81 81 81 139 139 139 F F F
96772024-02-15T03:01:33.976Z 23 460-479 83 86 86 143 148 148 F F F <---
96782024-02-15T03:01:33.976Z 24 480-499 81 86 86 139 148 148 F F F <---
96792024-02-15T03:01:33.976Z 25 500-519 84 84 84 145 145 145 F F F
96802024-02-15T03:01:33.976Z 26 520-539 82 82 82 141 141 141 F F F
96812024-02-15T03:01:33.976Z 27 540-559 82 82 82 140 140 140 F F F
96822024-02-15T03:01:33.976Z 28 560-579 84 84 84 145 145 145 F F F
96832024-02-15T03:01:33.976Z 29 580-599 82 82 82 142 142 142 F F F
96842024-02-15T03:01:33.977ZMax gen: 86, Max flush: 150
96852024-02-15T03:01:33.977ZError: Difference in extent metadata found!
96862024-02-15T03:01:33.977ZA Difference in extent metadata is expected here
96872024-02-15T03:01:33.977ZOn loop 43
96882024-02-15T03:01:33.977Z
96892024-02-15T03:01:33.977ZVerifying data now
96902024-02-15T03:01:39.192ZNo change after verify
96912024-02-15T03:01:39.192ZLoop: 43 Downstairs dump after verify (and repair):
96922024-02-15T03:01:39.741ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96932024-02-15T03:01:39.741Z 0 000-019 86 86 86 149 149 149 F F F
96942024-02-15T03:01:39.741Z 1 020-039 86 86 86 149 149 149 F F F
96952024-02-15T03:01:39.741Z 2 040-059 87 87 87 151 151 151 F F F
96962024-02-15T03:01:39.741Z 3 060-079 86 86 86 148 148 148 F F F
96972024-02-15T03:01:39.742Z 4 080-099 82 82 82 140 140 140 F F F
96982024-02-15T03:01:39.742Z 5 100-119 84 84 84 144 144 144 F F F
96992024-02-15T03:01:39.742Z 6 120-139 82 82 82 141 141 141 F F F
97002024-02-15T03:01:39.742Z 7 140-159 80 80 80 138 138 138 F F F
97012024-02-15T03:01:39.742Z 8 160-179 84 84 84 145 145 145 F F F
97022024-02-15T03:01:39.742Z 9 180-199 82 82 82 141 141 141 F F F
97032024-02-15T03:01:39.742Z 10 200-219 83 83 83 143 143 143 F F F
97042024-02-15T03:01:39.742Z 11 220-239 86 86 86 148 148 148 F F F
97052024-02-15T03:01:39.742Z 12 240-259 83 83 83 143 143 143 F F F
97062024-02-15T03:01:39.742Z 13 260-279 74 74 74 130 130 130 F F F
97072024-02-15T03:01:39.742Z 14 280-299 86 86 86 148 148 148 F F F
97082024-02-15T03:01:39.742Z 15 300-319 82 82 82 141 141 141 F F F
97092024-02-15T03:01:39.743Z 16 320-339 87 87 87 151 151 151 F F F
97102024-02-15T03:01:39.743Z 17 340-359 84 84 84 145 145 145 F F F
97112024-02-15T03:01:39.743Z 18 360-379 86 86 86 148 148 148 F F F
97122024-02-15T03:01:39.743Z 19 380-399 84 84 84 145 145 145 F F F
97132024-02-15T03:01:39.743Z 20 400-419 86 86 86 150 150 150 F F F
97142024-02-15T03:01:39.743Z 21 420-439 84 84 84 146 146 146 F F F
97152024-02-15T03:01:39.743Z 22 440-459 81 81 81 139 139 139 F F F
97162024-02-15T03:01:39.743Z 23 460-479 86 86 86 148 148 148 F F F
97172024-02-15T03:01:39.743Z 24 480-499 86 86 86 148 148 148 F F F
97182024-02-15T03:01:39.743Z 25 500-519 84 84 84 145 145 145 F F F
97192024-02-15T03:01:39.743Z 26 520-539 82 82 82 141 141 141 F F F
97202024-02-15T03:01:39.743Z 27 540-559 82 82 82 140 140 140 F F F
97212024-02-15T03:01:39.743Z 28 560-579 84 84 84 145 145 145 F F F
97222024-02-15T03:01:39.743Z 29 580-599 82 82 82 142 142 142 F F F
97232024-02-15T03:01:39.744ZMax gen: 87, Max flush: 151
97242024-02-15T03:01:39.757Z
97252024-02-15T03:01:39.757ZBegin loop 44
97262024-02-15T03:01:39.757ZDownstairs to restart: 0
97272024-02-15T03:01:39.757Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.141599039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487}
97282024-02-15T03:01:39.757Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.14190799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487}
97292024-02-15T03:01:39.757Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.141937336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487}
97302024-02-15T03:01:39.757Z{"msg":"Crucible 39604034-5d10-4769-996a-7a072090be4e has session id: ea5eaf8b-0bd6-4630-925e-9c97a8909b6d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142018716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97312024-02-15T03:01:39.757Z{"msg":"Crucible runtime is spawned
97322024-02-15T03:01:39.757ZUpstairs opts: Upstairs UUID: 39604034-5d10-4769-996a-7a072090be4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142078228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97332024-02-15T03:01:39.757Z{"msg":"Crucible stats registered with UUID: 39604034-5d10-4769-996a-7a072090be4e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142099496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97342024-02-15T03:01:39.757Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142178286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487}
97352024-02-15T03:01:39.757Z{"msg":"39604034-5d10-4769-996a-7a072090be4e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142262056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97362024-02-15T03:01:39.757Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142320108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97372024-02-15T03:01:39.757Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142351304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97382024-02-15T03:01:39.757Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142400408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97392024-02-15T03:01:39.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142429684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97402024-02-15T03:01:39.758Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.14246828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97412024-02-15T03:01:39.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142497096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97422024-02-15T03:01:39.758Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142535031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97432024-02-15T03:01:39.758Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.14262991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97442024-02-15T03:01:39.758Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142676014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97452024-02-15T03:01:39.758Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.14270654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97462024-02-15T03:01:39.758Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:38.142753664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97472024-02-15T03:01:39.758Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.14278464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97482024-02-15T03:01:39.758Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:38.142836013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97492024-02-15T03:01:39.758Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:38.142871779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97502024-02-15T03:01:39.758Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142895886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97512024-02-15T03:01:39.758Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.142929572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97522024-02-15T03:01:39.758Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.143065125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97532024-02-15T03:01:39.758Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.143471694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97542024-02-15T03:01:39.758Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.143514229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97552024-02-15T03:01:39.758Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.143540635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97562024-02-15T03:01:39.758Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.143603377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97572024-02-15T03:01:39.758Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144154468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97582024-02-15T03:01:39.758Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144273513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97592024-02-15T03:01:39.758Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144653216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97602024-02-15T03:01:39.759Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144704779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97612024-02-15T03:01:39.759Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144730956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97622024-02-15T03:01:39.759Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144760472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97632024-02-15T03:01:39.759Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144785929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97642024-02-15T03:01:39.759Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:38.144809776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97652024-02-15T03:01:40.223Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.144490689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97662024-02-15T03:01:40.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.144604715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97672024-02-15T03:01:40.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.145531499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97682024-02-15T03:01:40.239Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.145573104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97692024-02-15T03:01:40.239Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.146120875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97702024-02-15T03:01:40.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.146701152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97712024-02-15T03:01:40.239Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.146757225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97722024-02-15T03:01:40.239Z{"msg":"[0]R flush_numbers[0..12]: [149, 149, 151, 148, 140, 144, 141, 138, 145, 141, 143, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.146808149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97732024-02-15T03:01:40.239Z{"msg":"[0]R generation[0..12]: [86, 86, 87, 86, 82, 84, 82, 80, 84, 82, 83, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.146852033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97742024-02-15T03:01:40.239Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.146894368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97752024-02-15T03:01:40.240Z{"msg":"Wait for a query_work_queue command to finish before sending IO
97762024-02-15T03:01:40.240Z[1]R flush_numbers[0..12]: [149, 149, 151, 148, 140, 144, 141, 138, 145, 141, 143, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.146921525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97772024-02-15T03:01:40.240Z{"msg":"[1]R generation[0..12]: [86, 86, 87, 86, 82, 84, 82, 80, 84, 82, 83, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.14696174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97782024-02-15T03:01:40.240Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147001115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97792024-02-15T03:01:40.240Z{"msg":"[2]R flush_numbers[0..12]: [149, 149, 151, 148, 140, 144, 141, 138, 145, 141, 143, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147028471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97802024-02-15T03:01:40.240Z{"msg":"[2]R generation[0..12]: [86, 86, 87, 86, 82, 84, 82, 80, 84, 82, 83, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147065617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97812024-02-15T03:01:40.240Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147089394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97822024-02-15T03:01:40.240Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147126189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97832024-02-15T03:01:40.241Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147147336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97842024-02-15T03:01:40.241Z{"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147184012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97852024-02-15T03:01:40.241Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147209309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97862024-02-15T03:01:40.241Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147245134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97872024-02-15T03:01:40.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147263412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97882024-02-15T03:01:40.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147300927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97892024-02-15T03:01:40.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147323924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97902024-02-15T03:01:40.241Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.14735998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97912024-02-15T03:01:40.241Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147383667Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97922024-02-15T03:01:40.241Zip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97932024-02-15T03:01:40.241Z{"msg":"39604034-5d10-4769-996a-7a072090be4e is now active with session: ea5eaf8b-0bd6-4630-925e-9c97a8909b6d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147421752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97942024-02-15T03:01:40.242Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147456258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487,"session_id":"ea5eaf8b-0bd6-4630-925e-9c97a8909b6d"}
97952024-02-15T03:01:40.242Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:39.147475465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487}
97962024-02-15T03:01:40.242ZLoading write count information from file "/tmp/test_repair_verify.data"
97972024-02-15T03:01:40.242ZRun Repair workload
97982024-02-15T03:01:40.242Z01/30 Write block 114 len 8192 data: 5 7
97992024-02-15T03:01:40.242Z02/30 Read block 92 len 24576
98002024-02-15T03:01:40.252Z03/30 Write block 87 len 8192 data: 5 4
98012024-02-15T03:01:40.252Z04/30 Write block 512 len 4096 data: 11
98022024-02-15T03:01:40.253Z05/30 Flush
98032024-02-15T03:01:40.253Z06/30 Write block 537 len 28672 data: 8 7 7 6 7 9 9
98042024-02-15T03:01:40.253Z07/30 Write block 255 len 20480 data: 6 5 7 7 8
98052024-02-15T03:01:40.253Z08/30 Write block 25 len 40960 data: 3 5 6 7 6 7 7 6 7 7
98062024-02-15T03:01:40.253Z09/30 Write block 422 len 36864 data: 5 6 6 6 8 8 7 6 6
98072024-02-15T03:01:40.253Z10/30 Read block 479 len 40960
98082024-02-15T03:01:40.253Z11/30 Read block 98 len 32768
98092024-02-15T03:01:40.253Z12/30 Write block 575 len 32768 data: 11 12 12 9 7 8 7 8
98102024-02-15T03:01:40.253Z13/30 Write block 376 len 12288 data: 7 7 8
98112024-02-15T03:01:40.253Z14/30 Read block 423 len 16384
98122024-02-15T03:01:40.253Z15/30 Write block 194 len 28672 data: 10 11 10 9 7 6 7
98132024-02-15T03:01:40.253Z16/30 Read block 195 len 36864
98142024-02-15T03:01:40.254Z17/30 Read block 101 len 36864
98152024-02-15T03:01:40.254Z18/30 Read block 368 len 36864
98162024-02-15T03:01:40.254Z19/30 Read block 173 len 16384
98172024-02-15T03:01:40.258Z20/30 Read block 263 len 28672
98182024-02-15T03:01:40.258Z21/30 Write block 225 len 36864 data: 8 10 8 9 7 6 8 9 9
98192024-02-15T03:01:40.258Z22/30 Read block 489 len 12288
98202024-02-15T03:01:40.258Z23/30 Read block 69 len 40960
98212024-02-15T03:01:40.258Z24/30 Write block 15 len 20480 data: 4 4 5 7 6
98222024-02-15T03:01:40.259Z25/30 Write block 50 len 40960 data: 6 4 6 7 7 9 8 10 9 6
98232024-02-15T03:01:40.259Z26/30 Write block 303 len 36864 data: 12 12 13 10 9 8 9 10 10
98242024-02-15T03:01:40.259Z27/30 Read block 323 len 32768
98252024-02-15T03:01:40.259Z28/30 Read block 487 len 32768
98262024-02-15T03:01:40.259Z29/30 Read block 454 len 24576
98272024-02-15T03:01:40.259Z30/30 Write block 332 len 36864 data: 11 11 13 13 11 11 11 8 6
98282024-02-15T03:01:40.263Z----------------------------------------------------------------
98292024-02-15T03:01:40.263Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
98302024-02-15T03:01:40.263ZStates: Active Active Active
98312024-02-15T03:01:40.263ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
98322024-02-15T03:01:40.263Z 1 Acked 1000 Write 2 Done Done Done false
98332024-02-15T03:01:40.264Z 2 Acked 1001 Read 6 Done Done Done false
98342024-02-15T03:01:40.264Z 3 Acked 1002 Write 2 Done Done Done false
98352024-02-15T03:01:40.264Z 4 Acked 1003 Write 1 Sent Done Done false
98362024-02-15T03:01:40.264Z 5 Acked 1004 Flush 0 Sent Done Done false
98372024-02-15T03:01:40.264Z 6 Acked 1005 Write 7 Sent Done Done false
98382024-02-15T03:01:40.264Z 7 Acked 1006 Write 5 Sent Done Done false
98392024-02-15T03:01:40.264Z 8 Acked 1007 Write 10 Sent Done Done false
98402024-02-15T03:01:40.264Z 9 Acked 1008 Write 9 Sent Done Done false
98412024-02-15T03:01:40.264Z 10 Acked 1009 Read 10 Sent Done Done false
98422024-02-15T03:01:40.264Z 11 Acked 1010 Read 8 Sent Done Done false
98432024-02-15T03:01:40.264Z 12 Acked 1011 Write 8 Sent Done Done false
98442024-02-15T03:01:40.264Z 13 Acked 1012 Write 3 Sent Done Done false
98452024-02-15T03:01:40.265Z 14 Acked 1013 Read 4 Sent Done Done false
98462024-02-15T03:01:40.265Z 15 Acked 1014 Write 7 Sent Done Done false
98472024-02-15T03:01:40.265Z 16 Acked 1015 Read 9 Sent Done Done false
98482024-02-15T03:01:40.265Z 17 Acked 1016 Read 9 Sent Done Done false
98492024-02-15T03:01:40.265Z 18 Acked 1017 Read 9 Sent Done Done false
98502024-02-15T03:01:40.265Z 19 Acked 1018 Read 4 Sent Done Done false
98512024-02-15T03:01:40.265Z 20 Acked 1019 Read 7 Sent Done Done false
98522024-02-15T03:01:40.265Z 21 Acked 1020 Write 9 Sent Done Done false
98532024-02-15T03:01:40.265Z 22 Acked 1021 Read 3 Sent Done Done false
98542024-02-15T03:01:40.265Z 23 Acked 1022 Read 10 Sent Done Done false
98552024-02-15T03:01:40.265Z 24 Acked 1023 Write 5 Sent Done Done false
98562024-02-15T03:01:40.265Z 25 Acked 1024 Write 10 Sent Done Done false
98572024-02-15T03:01:40.266Z 26 Acked 1025 Write 9 Sent Done Done false
98582024-02-15T03:01:40.266Z 27 Acked 1026 Read 8 Sent Done Done false
98592024-02-15T03:01:40.266Z 28 Acked 1027 Read 8 Sent Done Done false
98602024-02-15T03:01:40.266Z 29 Acked 1028 Read 6 Sent Done Done false
98612024-02-15T03:01:40.266Z 30 Acked 1029 Write 9 Sent Sent Sent false
98622024-02-15T03:01:40.266Z STATES DS:0 DS:1 DS:2 TOTAL
98632024-02-15T03:01:40.266Z New 0 0 0 0
98642024-02-15T03:01:40.266Z Sent 27 1 1 29
98652024-02-15T03:01:40.266Z Done 3 29 29 61
98662024-02-15T03:01:40.266Z Skipped 0 0 0 0
98672024-02-15T03:01:40.266Z Error 0 0 0 0
98682024-02-15T03:01:40.267ZLast Flush: 0 1004 1004
98692024-02-15T03:01:40.267ZDownstairs last five completed:
98702024-02-15T03:01:40.267ZUpstairs last five completed: 30 29 28 27 26
98712024-02-15T03:01:40.267Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:39.179812344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1487}
98722024-02-15T03:01:40.267ZWrote out file "/tmp/test_repair_verify.data"
98732024-02-15T03:01:40.282Z
98742024-02-15T03:01:40.282ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98752024-02-15T03:01:40.297ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98762024-02-15T03:01:40.297Z 0 000-019 86 86 86 149 149 149 F T T <---
98772024-02-15T03:01:40.297Z 1 020-039 86 86 86 149 149 149 F T T <---
98782024-02-15T03:01:40.297Z 2 040-059 87 87 87 151 151 151 F T T <---
98792024-02-15T03:01:40.297Z 3 060-079 86 86 86 148 148 148 F F F
98802024-02-15T03:01:40.297Z 4 080-099 82 88 88 140 152 152 T F F <---
98812024-02-15T03:01:40.297Z 5 100-119 84 88 88 144 152 152 T F F <---
98822024-02-15T03:01:40.298Z 6 120-139 82 82 82 141 141 141 F F F
98832024-02-15T03:01:40.298Z 7 140-159 80 80 80 138 138 138 F F F
98842024-02-15T03:01:40.312Z 8 160-179 84 84 84 145 145 145 F F F
98852024-02-15T03:01:40.312Z 9 180-199 82 82 82 141 141 141 F T T <---
98862024-02-15T03:01:40.312Z 10 200-219 83 83 83 143 143 143 F T T <---
98872024-02-15T03:01:40.312Z 11 220-239 86 86 86 148 148 148 F T T <---
98882024-02-15T03:01:40.312Z 12 240-259 83 83 83 143 143 143 F T T <---
98892024-02-15T03:01:40.313Z 13 260-279 74 74 74 130 130 130 F F F
98902024-02-15T03:01:40.313Z 14 280-299 86 86 86 148 148 148 F F F
98912024-02-15T03:01:40.313Z 15 300-319 82 82 82 141 141 141 F T T <---
98922024-02-15T03:01:40.313Z 16 320-339 87 87 87 151 151 151 F T T <---
98932024-02-15T03:01:40.313Z 17 340-359 84 84 84 145 145 145 F T T <---
98942024-02-15T03:01:40.313Z 18 360-379 86 86 86 148 148 148 F T T <---
98952024-02-15T03:01:40.313Z 19 380-399 84 84 84 145 145 145 F F F
98962024-02-15T03:01:40.313Z 20 400-419 86 86 86 150 150 150 F F F
98972024-02-15T03:01:40.313Z 21 420-439 84 84 84 146 146 146 F T T <---
98982024-02-15T03:01:40.313Z 22 440-459 81 81 81 139 139 139 F F F
98992024-02-15T03:01:40.313Z 23 460-479 86 86 86 148 148 148 F F F
99002024-02-15T03:01:40.313Z 24 480-499 86 86 86 148 148 148 F F F
99012024-02-15T03:01:40.313Z 25 500-519 84 88 88 145 152 152 F F F <---
99022024-02-15T03:01:40.314Z 26 520-539 82 82 82 141 141 141 F T T <---
99032024-02-15T03:01:40.314Z 27 540-559 82 82 82 140 140 140 F T T <---
99042024-02-15T03:01:40.314Z 28 560-579 84 84 84 145 145 145 F T T <---
99052024-02-15T03:01:40.314Z 29 580-599 82 82 82 142 142 142 F T T <---
99062024-02-15T03:01:40.314ZMax gen: 88, Max flush: 152
99072024-02-15T03:01:40.314ZError: Difference in extent metadata found!
99082024-02-15T03:01:40.314ZA Difference in extent metadata is expected here
99092024-02-15T03:01:40.314ZOn loop 44
99102024-02-15T03:01:40.314Z
99112024-02-15T03:01:40.314ZVerifying data now
99122024-02-15T03:01:46.232ZNo change after verify
99132024-02-15T03:01:46.245ZLoop: 44 Downstairs dump after verify (and repair):
99142024-02-15T03:01:46.245ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99152024-02-15T03:01:46.245Z 0 000-019 89 89 89 153 153 153 F F F
99162024-02-15T03:01:46.245Z 1 020-039 89 89 89 153 153 153 F F F
99172024-02-15T03:01:46.245Z 2 040-059 89 89 89 153 153 153 F F F
99182024-02-15T03:01:46.246Z 3 060-079 86 86 86 148 148 148 F F F
99192024-02-15T03:01:46.246Z 4 080-099 88 88 88 152 152 152 F F F
99202024-02-15T03:01:46.246Z 5 100-119 88 88 88 152 152 152 F F F
99212024-02-15T03:01:46.246Z 6 120-139 82 82 82 141 141 141 F F F
99222024-02-15T03:01:46.246Z 7 140-159 80 80 80 138 138 138 F F F
99232024-02-15T03:01:46.246Z 8 160-179 84 84 84 145 145 145 F F F
99242024-02-15T03:01:46.246Z 9 180-199 89 89 89 153 153 153 F F F
99252024-02-15T03:01:46.247Z 10 200-219 89 89 89 153 153 153 F F F
99262024-02-15T03:01:46.247Z 11 220-239 89 89 89 153 153 153 F F F
99272024-02-15T03:01:46.247Z 12 240-259 89 89 89 153 153 153 F F F
99282024-02-15T03:01:46.247Z 13 260-279 74 74 74 130 130 130 F F F
99292024-02-15T03:01:46.247Z 14 280-299 86 86 86 148 148 148 F F F
99302024-02-15T03:01:46.247Z 15 300-319 89 89 89 153 153 153 F F F
99312024-02-15T03:01:46.248Z 16 320-339 89 89 89 153 153 153 F F F
99322024-02-15T03:01:46.248Z 17 340-359 89 89 89 153 153 153 F F F
99332024-02-15T03:01:46.249Z 18 360-379 89 89 89 153 153 153 F F F
99342024-02-15T03:01:46.249Z 19 380-399 84 84 84 145 145 145 F F F
99352024-02-15T03:01:46.249Z 20 400-419 86 86 86 150 150 150 F F F
99362024-02-15T03:01:46.249Z 21 420-439 89 89 89 153 153 153 F F F
99372024-02-15T03:01:46.250Z 22 440-459 81 81 81 139 139 139 F F F
99382024-02-15T03:01:46.250Z 23 460-479 86 86 86 148 148 148 F F F
99392024-02-15T03:01:46.250Z 24 480-499 86 86 86 148 148 148 F F F
99402024-02-15T03:01:46.250Z 25 500-519 88 88 88 152 152 152 F F F
99412024-02-15T03:01:46.250Z 26 520-539 89 89 89 153 153 153 F F F
99422024-02-15T03:01:46.250Z 27 540-559 89 89 89 153 153 153 F F F
99432024-02-15T03:01:46.251Z 28 560-579 89 89 89 153 153 153 F F F
99442024-02-15T03:01:46.251Z 29 580-599 89 89 89 153 153 153 F F F
99452024-02-15T03:01:46.251ZMax gen: 89, Max flush: 153
99462024-02-15T03:01:46.260Z
99472024-02-15T03:01:46.260ZBegin loop 45
99482024-02-15T03:01:46.261ZDownstairs to restart: 2
99492024-02-15T03:01:46.284Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.182764242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495}
99502024-02-15T03:01:46.284ZCrucible runtime is spawned
99512024-02-15T03:01:46.284Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183145544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495}
99522024-02-15T03:01:46.285Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183204587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495}
99532024-02-15T03:01:46.285Z{"msg":"Crucible 4a513df5-24e7-4b36-8487-65bb3e2e9bf2 has session id: 6f8ac958-584c-409a-a249-7d8898675132","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.18325455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99542024-02-15T03:01:46.285Z{"msg":"Upstairs opts: Upstairs UUID: 4a513df5-24e7-4b36-8487-65bb3e2e9bf2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183314923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99552024-02-15T03:01:46.285Z{"msg":"Crucible stats registered with UUID: 4a513df5-24e7-4b36-8487-65bb3e2e9bf2","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.1833355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99562024-02-15T03:01:46.285Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183362887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495}
99572024-02-15T03:01:46.285Z{"msg":"4a513df5-24e7-4b36-8487-65bb3e2e9bf2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183378355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99582024-02-15T03:01:46.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183394423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99592024-02-15T03:01:46.285Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183427469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99602024-02-15T03:01:46.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183453325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99612024-02-15T03:01:46.285Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183503549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99622024-02-15T03:01:46.285Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183536595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99632024-02-15T03:01:46.286Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183581909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99642024-02-15T03:01:46.286Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183626084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99652024-02-15T03:01:46.286Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183698495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99662024-02-15T03:01:46.286Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183744769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99672024-02-15T03:01:46.286Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183770976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99682024-02-15T03:01:46.286Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:45.18381725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99692024-02-15T03:01:46.286Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:45.183857735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99702024-02-15T03:01:46.286Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183879712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99712024-02-15T03:01:46.287Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:45.183969131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99722024-02-15T03:01:46.287Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.183996068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99732024-02-15T03:01:46.287Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.184010536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99742024-02-15T03:01:46.287Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.184094065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99752024-02-15T03:01:46.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.184495005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99762024-02-15T03:01:46.287Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.1845319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99772024-02-15T03:01:46.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.184590383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99782024-02-15T03:01:46.287Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.184668273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99792024-02-15T03:01:46.288Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.18517183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99802024-02-15T03:01:46.288Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.185218124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99812024-02-15T03:01:46.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.185757667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99822024-02-15T03:01:46.288Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.185800262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99832024-02-15T03:01:46.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.185825268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99842024-02-15T03:01:46.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.185865553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99852024-02-15T03:01:46.288Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.18589364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99862024-02-15T03:01:46.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:45.185917537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99872024-02-15T03:01:47.271Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.185724734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99882024-02-15T03:01:47.271Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.185815043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99892024-02-15T03:01:47.301Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.186734987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99902024-02-15T03:01:47.301Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.186773232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99912024-02-15T03:01:47.301Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.187260322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99922024-02-15T03:01:47.301Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.18782829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99932024-02-15T03:01:47.301Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.187868555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99942024-02-15T03:01:47.301Z{"msg":"[0]R flush_numbers[0..12]: [153, 153, 153, 148, 152, 152, 141, 138, 145, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.187897842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99952024-02-15T03:01:47.302Z{"msg":"[0]R generation[0..12]: [89, 89, 89, 86, 88, 88, 82, 80, 84, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.187935267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99962024-02-15T03:01:47.302Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.187958144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99972024-02-15T03:01:47.302Z{"msg":"[1]R flush_numbers[0..12]: [153, 153, 153, 148, 152, 152, 141, 138, 145, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.187980761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99982024-02-15T03:01:47.302Z{"msg":"[1]R generation[0..12]: [89, 89, 89, 86, 88, 88, 82, 80, 84, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.187995069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
99992024-02-15T03:01:47.303Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188009968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100002024-02-15T03:01:47.303Z{"msg":"[2]R flush_numbers[0..12]: [153, 153, 153, 148, 152, 152, 141, 138, 145, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188024486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100012024-02-15T03:01:47.303Z{"msg":"[2]R generation[0..12]: [89, 89, 89, 86, 88, 88, 82, 80, 84, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188038384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100022024-02-15T03:01:47.303Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188053112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100032024-02-15T03:01:47.303Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
100042024-02-15T03:01:47.303Z,"time":"2024-02-15T03:01:46.188066481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100052024-02-15T03:01:47.303Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188085568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100062024-02-15T03:01:47.303Z{"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188105816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100072024-02-15T03:01:47.303Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188119874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100082024-02-15T03:01:47.303Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188133892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100092024-02-15T03:01:47.304Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188159929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100102024-02-15T03:01:47.304Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188173727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100112024-02-15T03:01:47.304Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188195214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100122024-02-15T03:01:47.304Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188209012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100132024-02-15T03:01:47.304Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.18822925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100142024-02-15T03:01:47.304Z{"msg":"4a513df5-24e7-4b36-8487-65bb3e2e9bf2 is now active with session: 6f8ac958-584c-409a-a249-7d8898675132","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188244898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100152024-02-15T03:01:47.305Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188256897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495,"session_id":"6f8ac958-584c-409a-a249-7d8898675132"}
100162024-02-15T03:01:47.305Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:46.188269605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495}
100172024-02-15T03:01:47.305ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
100182024-02-15T03:01:47.305ZLoading write count information from file "/tmp/test_repair_verify.data"
100192024-02-15T03:01:47.305ZRun Repair workload
100202024-02-15T03:01:47.305Z01/30 Write block 98 len 24576 data: 11 9 9 10 10 7
100212024-02-15T03:01:47.305Z02/30 Write block 349 len 28672 data: 6 7 8 9 9 8 7
100222024-02-15T03:01:47.306Z03/30 Read block 503 len 32768
100232024-02-15T03:01:47.306Z04/30 Read block 160 len 36864
100242024-02-15T03:01:47.306Z05/30 Read block 485 len 4096
100252024-02-15T03:01:47.306Z06/30 Read block 208 len 28672
100262024-02-15T03:01:47.306Z07/30 Read block 112 len 32768
100272024-02-15T03:01:47.306Z08/30 Write block 86 len 8192 data: 5 6
100282024-02-15T03:01:47.306Z09/30 Read block 351 len 24576
100292024-02-15T03:01:47.306Z10/30 Read block 552 len 16384
100302024-02-15T03:01:47.306Z11/30 Flush
100312024-02-15T03:01:47.319Z12/30 Write block 349 len 12288 data: 7 8 9
100322024-02-15T03:01:47.320Z13/30 Read block 64 len 40960
100332024-02-15T03:01:47.320Z14/30 Flush
100342024-02-15T03:01:47.320Z15/30 Write block 204 len 24576 data: 6 5 6 6 7 6
100352024-02-15T03:01:47.320Z16/30 Read block 222 len 8192
100362024-02-15T03:01:47.320Z17/30 Read block 538 len 32768
100372024-02-15T03:01:47.320Z18/30 Read block 197 len 4096
100382024-02-15T03:01:47.320Z19/30 Read block 438 len 40960
100392024-02-15T03:01:47.320Z20/30 Read block 592 len 12288
100402024-02-15T03:01:47.321Z21/30 Read block 355 len 36864
100412024-02-15T03:01:47.321Z22/30 Read block 266 len 36864
100422024-02-15T03:01:47.321Z23/30 Read block 113 len 20480
100432024-02-15T03:01:47.321Z24/30 Write block 157 len 40960 data: 12 11 8 6 7 11 8 6 8 9
100442024-02-15T03:01:47.321Z25/30 Read block 225 len 36864
100452024-02-15T03:01:47.321Z26/30 Write block 558 len 36864 data: 9 10 9 7 7 7 8 8 4
100462024-02-15T03:01:47.321Z27/30 Read block 533 len 16384
100472024-02-15T03:01:47.321Z28/30 Write block 392 len 40960 data: 8 7 7 8 6 5 4 6 5 4
100482024-02-15T03:01:47.321Z29/30 Write block 583 len 32768 data: 6 7 6 7 5 4 7 9
100492024-02-15T03:01:47.321Z30/30 Write block 503 len 40960 data: 10 9 8 8 6 7 9 8 9 12
100502024-02-15T03:01:47.321Z----------------------------------------------------------------
100512024-02-15T03:01:47.322Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
100522024-02-15T03:01:47.322ZStates: Active Active Active
100532024-02-15T03:01:47.322ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
100542024-02-15T03:01:47.322Z 1 Acked 1000 Write 6 Done Done Done false
100552024-02-15T03:01:47.322Z 2 Acked 1001 Write 7 Done Done Done false
100562024-02-15T03:01:47.322Z 3 Acked 1002 Read 8 Done Done Done false
100572024-02-15T03:01:47.326Z 4 Acked 1003 Read 9 Done Done Sent false
100582024-02-15T03:01:47.326Z 5 Acked 1004 Read 1 Done Done Sent false
100592024-02-15T03:01:47.326Z 6 Acked 1005 Read 7 Done Done Sent false
100602024-02-15T03:01:47.326Z 7 Acked 1006 Read 8 Done Done Sent false
100612024-02-15T03:01:47.327Z 8 Acked 1007 Write 2 Done Done Sent false
100622024-02-15T03:01:47.327Z 9 Acked 1008 Read 6 Done Done Sent false
100632024-02-15T03:01:47.327Z 10 Acked 1009 Read 4 Done Done Sent false
100642024-02-15T03:01:47.327Z 11 Acked 1010 Flush 0 Done Done Sent false
100652024-02-15T03:01:47.327Z 12 Acked 1011 Write 3 Done Done Sent false
100662024-02-15T03:01:47.327Z 13 Acked 1012 Read 10 Done Done Sent false
100672024-02-15T03:01:47.339Z 14 Acked 1013 Flush 0 Done Done Sent false
100682024-02-15T03:01:47.339Z 15 Acked 1014 Write 6 Done Done Sent false
100692024-02-15T03:01:47.339Z 16 Acked 1015 Read 2 Done Done Sent false
100702024-02-15T03:01:47.339Z 17 Acked 1016 Read 8 Done Done Sent false
100712024-02-15T03:01:47.339Z 18 Acked 1017 Read 1 Done Done Sent false
100722024-02-15T03:01:47.339Z 19 Acked 1018 Read 10 Done Done Sent false
100732024-02-15T03:01:47.339Z 20 Acked 1019 Read 3 Done Done Sent false
100742024-02-15T03:01:47.339Z 21 Acked 1020 Read 9 Done Done Sent false
100752024-02-15T03:01:47.339Z 22 Acked 1021 Read 9 Done Done Sent false
100762024-02-15T03:01:47.339Z 23 Acked 1022 Read 5 Done Done Sent false
100772024-02-15T03:01:47.339Z 24 Acked 1023 Write 10 Done Done Sent false
100782024-02-15T03:01:47.339Z 25 Acked 1024 Read 9 Done Done Sent false
100792024-02-15T03:01:47.340Z 26 Acked 1025 Write 9 Done Done Sent false
100802024-02-15T03:01:47.340Z 27 Acked 1026 Read 4 Done Done Sent false
100812024-02-15T03:01:47.340Z 28 Acked 1027 Write 10 Done Done Sent false
100822024-02-15T03:01:47.340Z 29 Acked 1028 Write 8 Sent Sent Sent false
100832024-02-15T03:01:47.340Z 30 Acked 1029 Write 10 Sent Sent Sent false
100842024-02-15T03:01:47.340Z STATES DS:0 DS:1 DS:2 TOTAL
100852024-02-15T03:01:47.340Z New 0 0 0 0
100862024-02-15T03:01:47.340Z Sent 2 2 27 31
100872024-02-15T03:01:47.340Z Done 28 28 3 59
100882024-02-15T03:01:47.340Z Skipped 0 0 0 0
100892024-02-15T03:01:47.340Z Error 0 0 0 0
100902024-02-15T03:01:47.340ZLast Flush: 1013 1013 0
100912024-02-15T03:01:47.340ZDownstairs last five completed:
100922024-02-15T03:01:47.341ZUpstairs last five completed: 30 29 28 27 26
100932024-02-15T03:01:47.341Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:46.220164919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1495}
100942024-02-15T03:01:47.341ZWrote out file "/tmp/test_repair_verify.data"
100952024-02-15T03:01:47.341Z
100962024-02-15T03:01:47.341ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100972024-02-15T03:01:47.351ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100982024-02-15T03:01:47.351Z 0 000-019 89 89 89 153 153 153 F F F
100992024-02-15T03:01:47.351Z 1 020-039 89 89 89 153 153 153 F F F
101002024-02-15T03:01:47.351Z 2 040-059 89 89 89 153 153 153 F F F
101012024-02-15T03:01:47.351Z 3 060-079 86 86 86 148 148 148 F F F
101022024-02-15T03:01:47.352Z 4 080-099 90 90 88 154 154 152 F F T <---
101032024-02-15T03:01:47.352Z 5 100-119 90 90 88 154 154 152 F F T <---
101042024-02-15T03:01:47.352Z 6 120-139 82 82 82 141 141 141 F F F
101052024-02-15T03:01:47.352Z 7 140-159 80 80 80 138 138 138 T T F <---
101062024-02-15T03:01:47.352Z 8 160-179 84 84 84 145 145 145 T T F <---
101072024-02-15T03:01:47.352Z 9 180-199 89 89 89 153 153 153 F F F
101082024-02-15T03:01:47.352Z 10 200-219 89 89 89 153 153 153 T T F <---
101092024-02-15T03:01:47.353Z 11 220-239 89 89 89 153 153 153 F F F
101102024-02-15T03:01:47.353Z 12 240-259 89 89 89 153 153 153 F F F
101112024-02-15T03:01:47.353Z 13 260-279 74 74 74 130 130 130 F F F
101122024-02-15T03:01:47.353Z 14 280-299 86 86 86 148 148 148 F F F
101132024-02-15T03:01:47.353Z 15 300-319 89 89 89 153 153 153 F F F
101142024-02-15T03:01:47.353Z 16 320-339 89 89 89 153 153 153 F F F
101152024-02-15T03:01:47.353Z 17 340-359 90 90 89 155 155 153 F F T <---
101162024-02-15T03:01:47.353Z 18 360-379 89 89 89 153 153 153 F F F
101172024-02-15T03:01:47.353Z 19 380-399 84 84 84 145 145 145 T T F <---
101182024-02-15T03:01:47.353Z 20 400-419 86 86 86 150 150 150 T T F <---
101192024-02-15T03:01:47.354Z 21 420-439 89 89 89 153 153 153 F F F
101202024-02-15T03:01:47.354Z 22 440-459 81 81 81 139 139 139 F F F
101212024-02-15T03:01:47.354Z 23 460-479 86 86 86 148 148 148 F F F
101222024-02-15T03:01:47.354Z 24 480-499 86 86 86 148 148 148 F F F
101232024-02-15T03:01:47.354Z 25 500-519 88 88 88 152 152 152 T T F <---
101242024-02-15T03:01:47.354Z 26 520-539 89 89 89 153 153 153 F F F
101252024-02-15T03:01:47.354Z 27 540-559 89 89 89 153 153 153 T T F <---
101262024-02-15T03:01:47.354Z 28 560-579 89 89 89 153 153 153 T T F <---
101272024-02-15T03:01:47.354Z 29 580-599 89 89 89 153 153 153 T T F <---
101282024-02-15T03:01:47.354ZMax gen: 90, Max flush: 155
101292024-02-15T03:01:47.354ZError: Difference in extent metadata found!
101302024-02-15T03:01:47.354ZA Difference in extent metadata is expected here
101312024-02-15T03:01:47.354ZOn loop 45
101322024-02-15T03:01:47.354Z
101332024-02-15T03:01:47.354ZVerifying data now
101342024-02-15T03:01:52.972ZNo change after verify
101352024-02-15T03:01:52.972ZLoop: 45 Downstairs dump after verify (and repair):
101362024-02-15T03:01:52.980ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101372024-02-15T03:01:52.981Z 0 000-019 89 89 89 153 153 153 F F F
101382024-02-15T03:01:52.981Z 1 020-039 89 89 89 153 153 153 F F F
101392024-02-15T03:01:52.981Z 2 040-059 89 89 89 153 153 153 F F F
101402024-02-15T03:01:52.981Z 3 060-079 86 86 86 148 148 148 F F F
101412024-02-15T03:01:52.981Z 4 080-099 90 90 90 154 154 154 F F F
101422024-02-15T03:01:52.981Z 5 100-119 90 90 90 154 154 154 F F F
101432024-02-15T03:01:52.981Z 6 120-139 82 82 82 141 141 141 F F F
101442024-02-15T03:01:52.981Z 7 140-159 91 91 91 156 156 156 F F F
101452024-02-15T03:01:52.982Z 8 160-179 91 91 91 156 156 156 F F F
101462024-02-15T03:01:52.982Z 9 180-199 89 89 89 153 153 153 F F F
101472024-02-15T03:01:52.982Z 10 200-219 91 91 91 156 156 156 F F F
101482024-02-15T03:01:52.982Z 11 220-239 89 89 89 153 153 153 F F F
101492024-02-15T03:01:52.982Z 12 240-259 89 89 89 153 153 153 F F F
101502024-02-15T03:01:52.983Z 13 260-279 74 74 74 130 130 130 F F F
101512024-02-15T03:01:52.983Z 14 280-299 86 86 86 148 148 148 F F F
101522024-02-15T03:01:52.984Z 15 300-319 89 89 89 153 153 153 F F F
101532024-02-15T03:01:52.984Z 16 320-339 89 89 89 153 153 153 F F F
101542024-02-15T03:01:52.984Z 17 340-359 90 90 90 155 155 155 F F F
101552024-02-15T03:01:52.984Z 18 360-379 89 89 89 153 153 153 F F F
101562024-02-15T03:01:52.984Z 19 380-399 91 91 91 156 156 156 F F F
101572024-02-15T03:01:52.984Z 20 400-419 91 91 91 156 156 156 F F F
101582024-02-15T03:01:52.984Z 21 420-439 89 89 89 153 153 153 F F F
101592024-02-15T03:01:52.984Z 22 440-459 81 81 81 139 139 139 F F F
101602024-02-15T03:01:52.984Z 23 460-479 86 86 86 148 148 148 F F F
101612024-02-15T03:01:52.984Z 24 480-499 86 86 86 148 148 148 F F F
101622024-02-15T03:01:52.984Z 25 500-519 91 91 91 156 156 156 F F F
101632024-02-15T03:01:52.984Z 26 520-539 89 89 89 153 153 153 F F F
101642024-02-15T03:01:52.984Z 27 540-559 91 91 91 156 156 156 F F F
101652024-02-15T03:01:52.985Z 28 560-579 91 91 91 156 156 156 F F F
101662024-02-15T03:01:52.985Z 29 580-599 91 91 91 156 156 156 F F F
101672024-02-15T03:01:52.985ZMax gen: 91, Max flush: 156
101682024-02-15T03:01:52.985Z
101692024-02-15T03:01:52.985ZBegin loop 46
101702024-02-15T03:01:52.985ZDownstairs to restart: 2
101712024-02-15T03:01:53.009Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.914725747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503}
101722024-02-15T03:01:53.009ZCrucible runtime is spawned
101732024-02-15T03:01:53.055Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915143305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503}
101742024-02-15T03:01:53.056Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915200408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503}
101752024-02-15T03:01:53.056Z{"msg":"Crucible 477d0b37-16df-4ddf-aed2-ac0bcc885cc7 has session id: efcae530-c07f-453f-9447-823dcb96142b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915239363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101762024-02-15T03:01:53.056Z{"msg":"Upstairs opts: Upstairs UUID: 477d0b37-16df-4ddf-aed2-ac0bcc885cc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915288337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101772024-02-15T03:01:53.056Z{"msg":"Crucible stats registered with UUID: 477d0b37-16df-4ddf-aed2-ac0bcc885cc7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915320753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101782024-02-15T03:01:53.057Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.91534303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503}
101792024-02-15T03:01:53.057Z{"msg":"477d0b37-16df-4ddf-aed2-ac0bcc885cc7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915376945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101802024-02-15T03:01:53.057Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.91541997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101812024-02-15T03:01:53.058Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915447127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101822024-02-15T03:01:53.059Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915480173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101832024-02-15T03:01:53.059Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915505639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101842024-02-15T03:01:53.059Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915546414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101852024-02-15T03:01:53.060Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915575221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101862024-02-15T03:01:53.060Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915602227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101872024-02-15T03:01:53.060Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915641132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101882024-02-15T03:01:53.060Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915670239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101892024-02-15T03:01:53.061Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:51.915699985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101902024-02-15T03:01:53.061Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:51.915730301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101912024-02-15T03:01:53.061Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915756588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101922024-02-15T03:01:53.061Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:51.915786594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101932024-02-15T03:01:53.061Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915836588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101942024-02-15T03:01:53.061Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915865294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101952024-02-15T03:01:53.062Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.91589828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101962024-02-15T03:01:53.062Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.915932966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101972024-02-15T03:01:53.062Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.916398947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101982024-02-15T03:01:53.062Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.916432013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
101992024-02-15T03:01:53.062Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.91645761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102002024-02-15T03:01:53.062Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.916500495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102012024-02-15T03:01:53.062Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.916998182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102022024-02-15T03:01:53.062Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.917153573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102032024-02-15T03:01:53.062Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.91757549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102042024-02-15T03:01:53.062Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.917620604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102052024-02-15T03:01:53.063Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.917642371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102062024-02-15T03:01:53.063Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.917671168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102072024-02-15T03:01:53.063Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.917694685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102082024-02-15T03:01:53.063Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:51.917709483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102092024-02-15T03:01:54.003Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.917753111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102102024-02-15T03:01:54.445Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.918011758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102112024-02-15T03:01:54.445Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.918818957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102122024-02-15T03:01:54.445Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.91887206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102132024-02-15T03:01:54.446Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.919372908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102142024-02-15T03:01:54.446Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.919939777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102152024-02-15T03:01:54.446Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.919975142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102162024-02-15T03:01:54.446Z{"msg":"[0]R flush_numbers[0..12]: [153, 153, 153, 148, 154, 154, 141, 156, 156, 153, 156, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.91999518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102172024-02-15T03:01:54.446Z{"msg":"[0]R generation[0..12]: [89, 89, 89, 86, 90, 90, 82, 91, 91, 89, 91, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920030905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102182024-02-15T03:01:54.447Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920058982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102192024-02-15T03:01:54.447Z{"msg":"[1]R flush_numbers[0..12]: [153, 153, 153, 148, 154, 154, 141, 156, 156, 153, 156, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920077849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102202024-02-15T03:01:54.447Z{"msg":"[1]R generation[0..12]: [89, 89, 89, 86, 90, 90, 82, 91, 91, 89, 91, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920095427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102212024-02-15T03:01:54.447Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920122704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102222024-02-15T03:01:54.447Z{"msg":"[2]R flush_numbers[0..12]: [153, 153, 153, 148, 154, 154, 141, 156, 156, 153, 156, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920141132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102232024-02-15T03:01:54.447ZWait for a query_work_queue command to finish before sending IO
102242024-02-15T03:01:54.448Z{"msg":"[2]R generation[0..12]: [89, 89, 89, 86, 90, 90, 82, 91, 91, 89, 91, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920169558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102252024-02-15T03:01:54.448Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920188815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102262024-02-15T03:01:54.448Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920213383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102272024-02-15T03:01:54.448Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.92023125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102282024-02-15T03:01:54.448Z{"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920256017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102292024-02-15T03:01:54.448Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920274075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102302024-02-15T03:01:54.448Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920298792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102312024-02-15T03:01:54.448Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.92031439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102322024-02-15T03:01:54.448Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920339067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102332024-02-15T03:01:54.448Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920355845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102342024-02-15T03:01:54.448Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920380731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102352024-02-15T03:01:54.448Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.92039722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102362024-02-15T03:01:54.448Z{"msg":"477d0b37-16df-4ddf-aed2-ac0bcc885cc7 is now active with session: efcae530-c07f-453f-9447-823dcb96142b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920413497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102372024-02-15T03:01:54.448Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920428536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503,"session_id":"efcae530-c07f-453f-9447-823dcb96142b"}
102382024-02-15T03:01:54.448Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:52.920451853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503}
102392024-02-15T03:01:54.448ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
102402024-02-15T03:01:54.449ZLoading write count information from file "/tmp/test_repair_verify.data"
102412024-02-15T03:01:54.449ZRun Repair workload
102422024-02-15T03:01:54.449Z01/30 Write block 214 len 12288 data: 10 9 8
102432024-02-15T03:01:54.449Z02/30 Read block 441 len 8192
102442024-02-15T03:01:54.449Z03/30 Read block 509 len 16384
102452024-02-15T03:01:54.449Z04/30 Write block 31 len 32768 data: 8 7 8 8 7 6 6 7
102462024-02-15T03:01:54.449Z05/30 Read block 548 len 16384
102472024-02-15T03:01:54.449Z06/30 Read block 283 len 12288
102482024-02-15T03:01:54.449Z07/30 Write block 234 len 40960 data: 10 9 9 7 6 6 7 8 8 7
102492024-02-15T03:01:54.449Z08/30 Read block 179 len 4096
102502024-02-15T03:01:54.449Z09/30 Read block 373 len 12288
102512024-02-15T03:01:54.449Z10/30 Write block 117 len 20480 data: 8 7 6 6 7
102522024-02-15T03:01:54.449Z11/30 Write block 104 len 16384 data: 8 8 5 4
102532024-02-15T03:01:54.450Z12/30 Read block 106 len 28672
102542024-02-15T03:01:54.450Z13/30 Write block 53 len 12288 data: 8 8 10
102552024-02-15T03:01:54.450Z14/30 Flush
102562024-02-15T03:01:54.450Z15/30 Write block 273 len 40960 data: 7 5 6 6 8 5 5 5 5 4
102572024-02-15T03:01:54.450Z16/30 Read block 77 len 40960
102582024-02-15T03:01:54.450Z17/30 Read block 218 len 20480
102592024-02-15T03:01:54.450Z18/30 Write block 96 len 16384 data: 9 11 12 10
102602024-02-15T03:01:54.450Z19/30 Read block 534 len 40960
102612024-02-15T03:01:54.450Z20/30 Read block 497 len 20480
102622024-02-15T03:01:54.450Z21/30 Write block 358 len 8192 data: 9 9
102632024-02-15T03:01:54.450Z22/30 Write block 163 len 28672 data: 9 7 9 10 8 7 6
102642024-02-15T03:01:54.450Z23/30 Write block 470 len 32768 data: 10 10 8 7 6 5 6 8
102652024-02-15T03:01:54.451Z24/30 Flush
102662024-02-15T03:01:54.451Z25/30 Write block 122 len 36864 data: 6 6 6 5 6 6 5 5 5
102672024-02-15T03:01:54.451Z26/30 Write block 585 len 36864 data: 7 8 6 5 8 10 12 10 11
102682024-02-15T03:01:54.451Z27/30 Write block 385 len 12288 data: 10 9 9
102692024-02-15T03:01:54.451Z28/30 Read block 371 len 40960
102702024-02-15T03:01:54.451Z29/30 Read block 135 len 36864
102712024-02-15T03:01:54.451Z30/30 Write block 557 len 16384 data: 9 10 11 10
102722024-02-15T03:01:54.451Z----------------------------------------------------------------
102732024-02-15T03:01:54.451Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
102742024-02-15T03:01:54.451ZStates: Active Active Active
102752024-02-15T03:01:54.451ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102762024-02-15T03:01:54.451Z 1 Acked 1000 Write 3 Done Done Done false
102772024-02-15T03:01:54.452Z 2 Acked 1001 Read 2 Done Done Done false
102782024-02-15T03:01:54.452Z 3 Acked 1002 Read 4 Done Done Done false
102792024-02-15T03:01:54.452Z 4 Acked 1003 Write 8 Done Done Done false
102802024-02-15T03:01:54.452Z 5 Acked 1004 Read 4 Done Done Done false
102812024-02-15T03:01:54.452Z 6 Acked 1005 Read 3 Done Done Sent false
102822024-02-15T03:01:54.452Z 7 Acked 1006 Write 10 Done Done Sent false
102832024-02-15T03:01:54.452Z 8 Acked 1007 Read 1 Done Done Sent false
102842024-02-15T03:01:54.452Z 9 Acked 1008 Read 3 Done Done Sent false
102852024-02-15T03:01:54.453Z 10 Acked 1009 Write 5 Done Done Sent false
102862024-02-15T03:01:54.453Z 11 Acked 1010 Write 4 Done Done Sent false
102872024-02-15T03:01:54.453Z 12 Acked 1011 Read 7 Done Done Sent false
102882024-02-15T03:01:54.453Z 13 Acked 1012 Write 3 Done Done Sent false
102892024-02-15T03:01:54.453Z 14 Acked 1013 Flush 0 Done Done Sent false
102902024-02-15T03:01:54.453Z 15 Acked 1014 Write 10 Done Done Sent false
102912024-02-15T03:01:54.453Z 16 Acked 1015 Read 10 Done Done Sent false
102922024-02-15T03:01:54.453Z 17 Acked 1016 Read 5 Done Done Sent false
102932024-02-15T03:01:54.453Z 18 Acked 1017 Write 4 Done Done Sent false
102942024-02-15T03:01:54.453Z 19 Acked 1018 Read 10 Done Done Sent false
102952024-02-15T03:01:54.453Z 20 Acked 1019 Read 5 Done Done Sent false
102962024-02-15T03:01:54.454Z 21 Acked 1020 Write 2 Done Done Sent false
102972024-02-15T03:01:54.454Z 22 Acked 1021 Write 7 Done Done Sent false
102982024-02-15T03:01:54.454Z 23 Acked 1022 Write 8 Done Done Sent false
102992024-02-15T03:01:54.454Z 24 Acked 1023 Flush 0 Done Done Sent false
103002024-02-15T03:01:54.454Z 25 Acked 1024 Write 9 Done Done Sent false
103012024-02-15T03:01:54.454Z 26 Acked 1025 Write 9 Done Done Sent false
103022024-02-15T03:01:54.454Z 27 Acked 1026 Write 3 Done Done Sent false
103032024-02-15T03:01:54.454Z 28 Acked 1027 Read 10 Done Done Sent false
103042024-02-15T03:01:54.455Z 29 Acked 1028 Read 9 Done Done Sent false
103052024-02-15T03:01:54.455Z 30 Acked 1029 Write 4 Sent Sent Sent false
103062024-02-15T03:01:54.455Z STATES DS:0 DS:1 DS:2 TOTAL
103072024-02-15T03:01:54.455Z New 0 0 0 0
103082024-02-15T03:01:54.455Z Sent 1 1 25 27
103092024-02-15T03:01:54.455Z Done 29 29 5 63
103102024-02-15T03:01:54.467Z Skipped 0 0 0 0
103112024-02-15T03:01:54.467Z Error 0 0 0 0
103122024-02-15T03:01:54.467ZLast Flush: 1023 1023 0
103132024-02-15T03:01:54.467ZDownstairs last five completed:
103142024-02-15T03:01:54.467ZUpstairs last five completed: 30 29 28 27 26
103152024-02-15T03:01:54.467Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:52.953023292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1503}
103162024-02-15T03:01:54.467ZWrote out file "/tmp/test_repair_verify.data"
103172024-02-15T03:01:54.467Z
103182024-02-15T03:01:54.467ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
103192024-02-15T03:01:54.468ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103202024-02-15T03:01:54.468Z 0 000-019 89 89 89 153 153 153 F F F
103212024-02-15T03:01:54.468Z 1 020-039 92 92 89 157 157 153 F F T <---
103222024-02-15T03:01:54.468Z 2 040-059 92 92 89 157 157 153 F F F <---
103232024-02-15T03:01:54.468Z 3 060-079 86 86 86 148 148 148 F F F
103242024-02-15T03:01:54.468Z 4 080-099 92 92 90 158 158 154 F F F <---
103252024-02-15T03:01:54.468Z 5 100-119 92 92 90 157 157 154 F F F <---
103262024-02-15T03:01:54.468Z 6 120-139 92 92 82 157 157 141 T T F <---
103272024-02-15T03:01:54.468Z 7 140-159 91 91 91 156 156 156 F F F
103282024-02-15T03:01:54.468Z 8 160-179 92 92 91 158 158 156 F F F <---
103292024-02-15T03:01:54.468Z 9 180-199 89 89 89 153 153 153 F F F
103302024-02-15T03:01:54.468Z 10 200-219 92 92 91 157 157 156 F F T <---
103312024-02-15T03:01:54.468Z 11 220-239 92 92 89 157 157 153 F F F <---
103322024-02-15T03:01:54.469Z 12 240-259 92 92 89 157 157 153 F F F <---
103332024-02-15T03:01:54.469Z 13 260-279 92 92 74 158 158 130 F F F <---
103342024-02-15T03:01:54.469Z 14 280-299 92 92 86 158 158 148 F F F <---
103352024-02-15T03:01:54.469Z 15 300-319 89 89 89 153 153 153 F F F
103362024-02-15T03:01:54.469Z 16 320-339 89 89 89 153 153 153 F F F
103372024-02-15T03:01:54.469Z 17 340-359 92 92 90 158 158 155 F F F <---
103382024-02-15T03:01:54.469Z 18 360-379 89 89 89 153 153 153 F F F
103392024-02-15T03:01:54.469Z 19 380-399 91 91 91 156 156 156 T T F <---
103402024-02-15T03:01:54.469Z 20 400-419 91 91 91 156 156 156 F F F
103412024-02-15T03:01:54.469Z 21 420-439 89 89 89 153 153 153 F F F
103422024-02-15T03:01:54.469Z 22 440-459 81 81 81 139 139 139 F F F
103432024-02-15T03:01:54.469Z 23 460-479 92 92 86 158 158 148 F F F <---
103442024-02-15T03:01:54.469Z 24 480-499 86 86 86 148 148 148 F F F
103452024-02-15T03:01:54.469Z 25 500-519 91 91 91 156 156 156 F F F
103462024-02-15T03:01:54.469Z 26 520-539 89 89 89 153 153 153 F F F
103472024-02-15T03:01:54.469Z 27 540-559 91 91 91 156 156 156 T T F <---
103482024-02-15T03:01:54.469Z 28 560-579 91 91 91 156 156 156 T T F <---
103492024-02-15T03:01:54.469Z 29 580-599 91 91 91 156 156 156 T T F <---
103502024-02-15T03:01:54.469ZMax gen: 92, Max flush: 158
103512024-02-15T03:01:54.469ZError: Difference in extent metadata found!
103522024-02-15T03:01:54.469ZA Difference in extent metadata is expected here
103532024-02-15T03:01:54.470ZOn loop 46
103542024-02-15T03:01:54.470Z
103552024-02-15T03:01:54.470ZVerifying data now
103562024-02-15T03:01:59.889ZNo change after verify
103572024-02-15T03:01:59.889ZLoop: 46 Downstairs dump after verify (and repair):
103582024-02-15T03:01:59.905ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103592024-02-15T03:01:59.905Z 0 000-019 89 89 89 153 153 153 F F F
103602024-02-15T03:01:59.906Z 1 020-039 92 92 92 157 157 157 F F F
103612024-02-15T03:01:59.906Z 2 040-059 92 92 92 157 157 157 F F F
103622024-02-15T03:01:59.906Z 3 060-079 86 86 86 148 148 148 F F F
103632024-02-15T03:01:59.906Z 4 080-099 92 92 92 158 158 158 F F F
103642024-02-15T03:01:59.906Z 5 100-119 92 92 92 157 157 157 F F F
103652024-02-15T03:01:59.906Z 6 120-139 93 93 93 159 159 159 F F F
103662024-02-15T03:01:59.906Z 7 140-159 91 91 91 156 156 156 F F F
103672024-02-15T03:01:59.906Z 8 160-179 92 92 92 158 158 158 F F F
103682024-02-15T03:01:59.906Z 9 180-199 89 89 89 153 153 153 F F F
103692024-02-15T03:01:59.906Z 10 200-219 92 92 92 157 157 157 F F F
103702024-02-15T03:01:59.906Z 11 220-239 92 92 92 157 157 157 F F F
103712024-02-15T03:01:59.906Z 12 240-259 92 92 92 157 157 157 F F F
103722024-02-15T03:01:59.915Z 13 260-279 92 92 92 158 158 158 F F F
103732024-02-15T03:01:59.915Z 14 280-299 92 92 92 158 158 158 F F F
103742024-02-15T03:01:59.915Z 15 300-319 89 89 89 153 153 153 F F F
103752024-02-15T03:01:59.915Z 16 320-339 89 89 89 153 153 153 F F F
103762024-02-15T03:01:59.915Z 17 340-359 92 92 92 158 158 158 F F F
103772024-02-15T03:01:59.915Z 18 360-379 89 89 89 153 153 153 F F F
103782024-02-15T03:01:59.916Z 19 380-399 93 93 93 159 159 159 F F F
103792024-02-15T03:01:59.916Z 20 400-419 91 91 91 156 156 156 F F F
103802024-02-15T03:01:59.916Z 21 420-439 89 89 89 153 153 153 F F F
103812024-02-15T03:01:59.916Z 22 440-459 81 81 81 139 139 139 F F F
103822024-02-15T03:01:59.916Z 23 460-479 92 92 92 158 158 158 F F F
103832024-02-15T03:01:59.916Z 24 480-499 86 86 86 148 148 148 F F F
103842024-02-15T03:01:59.916Z 25 500-519 91 91 91 156 156 156 F F F
103852024-02-15T03:01:59.916Z 26 520-539 89 89 89 153 153 153 F F F
103862024-02-15T03:01:59.917Z 27 540-559 93 93 93 159 159 159 F F F
103872024-02-15T03:01:59.917Z 28 560-579 93 93 93 159 159 159 F F F
103882024-02-15T03:01:59.917Z 29 580-599 93 93 93 159 159 159 F F F
103892024-02-15T03:01:59.917ZMax gen: 93, Max flush: 159
103902024-02-15T03:01:59.917Z
103912024-02-15T03:01:59.917ZBegin loop 47
103922024-02-15T03:01:59.917ZDownstairs to restart: 0
103932024-02-15T03:01:59.976Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.847438529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511}
103942024-02-15T03:01:59.976Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
103952024-02-15T03:01:59.976Z,"time":"2024-02-15T03:01:58.847789155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511}
103962024-02-15T03:01:59.976Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.84782655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511}
103972024-02-15T03:01:59.976Z{"msg":"Crucible ab187287-67a0-4539-91e8-cb9581587592 has session id: 1881bfab-1295-4e17-b983-d5d1fd7e18ec","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.847890292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
103982024-02-15T03:01:59.977Z{"msg":"Upstairs opts: Upstairs UUID: ab187287-67a0-4539-91e8-cb9581587592, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.847955134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
103992024-02-15T03:01:59.977Z{"msg":"Crucible stats registered with UUID: ab187287-67a0-4539-91e8-cb9581587592","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.847998529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104002024-02-15T03:01:59.977Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848017326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511}
104012024-02-15T03:01:59.977Z{"msg":"ab187287-67a0-4539-91e8-cb9581587592 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848037394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104022024-02-15T03:01:59.977Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.84806333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104032024-02-15T03:01:59.977Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848082498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104042024-02-15T03:01:59.977Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848099996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104052024-02-15T03:01:59.978Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848125963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104062024-02-15T03:01:59.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848174347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104072024-02-15T03:01:59.978Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.84822465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104082024-02-15T03:01:59.978Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848264205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104092024-02-15T03:01:59.978Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848314949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104102024-02-15T03:01:59.978Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848343026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104112024-02-15T03:01:59.978Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848404758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104122024-02-15T03:01:59.978Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848445602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104132024-02-15T03:01:59.978Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:58.848692082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104142024-02-15T03:01:59.978Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:58.848756574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104152024-02-15T03:01:59.978Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:58.848829005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104162024-02-15T03:01:59.978Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848870009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104172024-02-15T03:01:59.979Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.848897406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104182024-02-15T03:01:59.979Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.849009392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104192024-02-15T03:01:59.979Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.849179731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104202024-02-15T03:01:59.979Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.849235584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104212024-02-15T03:01:59.979Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.849284268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104222024-02-15T03:01:59.979Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.849321053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104232024-02-15T03:01:59.996Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.849812961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104242024-02-15T03:01:59.996Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.849917488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104252024-02-15T03:01:59.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.850428804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104262024-02-15T03:01:59.996Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.85046492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104272024-02-15T03:01:59.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.850481418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104282024-02-15T03:01:59.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.850494216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104292024-02-15T03:01:59.996Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.850509254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104302024-02-15T03:01:59.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:58.850531911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104312024-02-15T03:02:01.345Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.850215874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104322024-02-15T03:02:01.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.85040458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104332024-02-15T03:02:01.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.851559266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104342024-02-15T03:02:01.382Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.85160637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104352024-02-15T03:02:01.382Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852104827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104362024-02-15T03:02:01.382Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852676775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104372024-02-15T03:02:01.382Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852710841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104382024-02-15T03:02:01.382Z{"msg":"[0]R flush_numbers[0..12]: [153, 157, 157, 148, 158, 157, 159, 156, 158, 153, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852726299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104392024-02-15T03:02:01.382Z{"msg":"[0]R generation[0..12]: [89, 92, 92, 86, 92, 92, 93, 91, 92, 89, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852741238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104402024-02-15T03:02:01.382Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852763855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104412024-02-15T03:02:01.382Z{"msg":"[1]R flush_numbers[0..12]: [153, 157, 157, 148, 158, 157, 159, 156, 158, 153, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852786282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104422024-02-15T03:02:01.382Z{"msg":"[1]R generation[0..12]: [89, 92, 92, 86, 92, 92, 93, 91, 92, 89, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.85280048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104432024-02-15T03:02:01.382Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852814998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104442024-02-15T03:02:01.383Z{"msg":"[2]R flush_numbers[0..12]: [153, 157, 157, 148, 158, 157, 159, 156, 158, 153, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852836156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104452024-02-15T03:02:01.383Z{"msg":"[2]R generation[0..12]: [89, 92, 92, 86, 92, 92, 93, 91, 92, 89, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852850434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104462024-02-15T03:02:01.383Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852865222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104472024-02-15T03:02:01.383Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.85287864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104482024-02-15T03:02:01.383Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852894208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104492024-02-15T03:02:01.383Z{"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852907607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104502024-02-15T03:02:01.383Z{"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
104512024-02-15T03:02:01.383Z,"time":"2024-02-15T03:01:59.852920945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104522024-02-15T03:02:01.383Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852948392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104532024-02-15T03:02:01.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.8529607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104542024-02-15T03:02:01.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852982097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104552024-02-15T03:02:01.383Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.852995766Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104562024-02-15T03:02:01.383Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.853009384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104572024-02-15T03:02:01.383Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.853021772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104582024-02-15T03:02:01.383Z{"msg":"ab187287-67a0-4539-91e8-cb9581587592 is now active with session: 1881bfab-1295-4e17-b983-d5d1fd7e18ec","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.853035801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104592024-02-15T03:02:01.383Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.853047799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511,"session_id":"1881bfab-1295-4e17-b983-d5d1fd7e18ec"}
104602024-02-15T03:02:01.383Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:01:59.853060667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511}
104612024-02-15T03:02:01.383ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
104622024-02-15T03:02:01.383ZLoading write count information from file "/tmp/test_repair_verify.data"
104632024-02-15T03:02:01.383ZRun Repair workload
104642024-02-15T03:02:01.383Z01/30 Flush
104652024-02-15T03:02:01.383Z02/30 Write block 103 len 8192 data: 8 9
104662024-02-15T03:02:01.383Z03/30 Write block 295 len 20480 data: 9 8 6 7 8
104672024-02-15T03:02:01.383Z04/30 Write block 120 len 8192 data: 7 8
104682024-02-15T03:02:01.383Z05/30 Write block 436 len 12288 data: 8 9 8
104692024-02-15T03:02:01.384Z06/30 Write block 539 len 4096 data: 8
104702024-02-15T03:02:01.384Z07/30 Write block 582 len 8192 data: 9 7
104712024-02-15T03:02:01.384Z08/30 Write block 314 len 40960 data: 7 6 7 7 7 6 5 5 5 7
104722024-02-15T03:02:01.384Z09/30 Write block 284 len 8192 data: 10 12
104732024-02-15T03:02:01.384Z10/30 Write block 49 len 16384 data: 9 7 5 7
104742024-02-15T03:02:01.384Z11/30 Read block 175 len 8192
104752024-02-15T03:02:01.384Z12/30 Read block 523 len 4096
104762024-02-15T03:02:01.384Z13/30 Write block 84 len 12288 data: 6 6 6
104772024-02-15T03:02:01.384Z14/30 Write block 438 len 8192 data: 9 7
104782024-02-15T03:02:01.384Z15/30 Write block 405 len 16384 data: 8 9 10 11
104792024-02-15T03:02:01.384Z16/30 Read block 205 len 20480
104802024-02-15T03:02:01.384Z17/30 Write block 3 len 8192 data: 5 7
104812024-02-15T03:02:01.384Z18/30 Read block 485 len 20480
104822024-02-15T03:02:01.384Z19/30 Write block 384 len 24576 data: 13 11 10 10 9 10
104832024-02-15T03:02:01.384Z20/30 Write block 572 len 12288 data: 7 10 11
104842024-02-15T03:02:01.384Z21/30 Read block 576 len 16384
104852024-02-15T03:02:01.384Z22/30 Read block 425 len 32768
104862024-02-15T03:02:01.384Z23/30 Write block 58 len 28672 data: 10 7 7 7 8 8 8
104872024-02-15T03:02:01.384Z24/30 Read block 535 len 24576
104882024-02-15T03:02:01.384Z25/30 Read block 540 len 36864
104892024-02-15T03:02:01.384Z26/30 Write block 580 len 16384 data: 9 8 10 8
104902024-02-15T03:02:01.384Z27/30 Read block 586 len 32768
104912024-02-15T03:02:01.384Z28/30 Write block 546 len 16384 data: 6 6 6 5
104922024-02-15T03:02:01.385Z29/30 Write block 479 len 16384 data: 11 9 9 8
104932024-02-15T03:02:01.385Z30/30 Write block 487 len 12288 data: 8 6 8
104942024-02-15T03:02:01.385Z----------------------------------------------------------------
104952024-02-15T03:02:01.385Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:29
104962024-02-15T03:02:01.385ZStates: Active Active Active
104972024-02-15T03:02:01.385ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104982024-02-15T03:02:01.385Z 2 Acked 1001 Write 2 Done Done Done false
104992024-02-15T03:02:01.385Z 3 Acked 1002 Write 5 Done Done Done false
105002024-02-15T03:02:01.385Z 4 Acked 1003 Write 2 Sent Done Done false
105012024-02-15T03:02:01.385Z 5 Acked 1004 Write 3 Sent Done Done false
105022024-02-15T03:02:01.385Z 6 Acked 1005 Write 1 Sent Done Done false
105032024-02-15T03:02:01.385Z 7 Acked 1006 Write 2 Sent Done Done false
105042024-02-15T03:02:01.385Z 8 Acked 1007 Write 10 Sent Done Done false
105052024-02-15T03:02:01.385Z 9 Acked 1008 Write 2 Sent Done Done false
105062024-02-15T03:02:01.385Z 10 Acked 1009 Write 4 Sent Done Done false
105072024-02-15T03:02:01.385Z 11 Acked 1010 Read 2 Sent Done Done false
105082024-02-15T03:02:01.385Z 12 Acked 1011 Read 1 Sent Done Done false
105092024-02-15T03:02:01.385Z 13 Acked 1012 Write 3 Sent Done Done false
105102024-02-15T03:02:01.385Z 14 Acked 1013 Write 2 Sent Done Done false
105112024-02-15T03:02:01.385Z 15 Acked 1014 Write 4 Sent Done Done false
105122024-02-15T03:02:01.385Z 16 Acked 1015 Read 5 Sent Done Done false
105132024-02-15T03:02:01.385Z 17 Acked 1016 Write 2 Sent Done Done false
105142024-02-15T03:02:01.385Z 18 Acked 1017 Read 5 Sent Done Done false
105152024-02-15T03:02:01.385Z 19 Acked 1018 Write 6 Sent Done Done false
105162024-02-15T03:02:01.385Z 20 Acked 1019 Write 3 Sent Done Done false
105172024-02-15T03:02:01.385Z 21 Acked 1020 Read 4 Sent Done Done false
105182024-02-15T03:02:01.386Z 22 Acked 1021 Read 8 Sent Done Done false
105192024-02-15T03:02:01.386Z 23 Acked 1022 Write 7 Sent Done Done false
105202024-02-15T03:02:01.386Z 24 Acked 1023 Read 6 Sent Done Done false
105212024-02-15T03:02:01.386Z 25 Acked 1024 Read 9 Sent Done Done false
105222024-02-15T03:02:01.386Z 26 Acked 1025 Write 4 Sent Done Done false
105232024-02-15T03:02:01.386Z 27 Acked 1026 Read 8 Sent Done Done false
105242024-02-15T03:02:01.386Z 28 Acked 1027 Write 4 Sent Done Done false
105252024-02-15T03:02:01.386Z 29 Acked 1028 Write 4 Sent Sent Sent false
105262024-02-15T03:02:01.386Z 30 Acked 1029 Write 3 Sent Sent Sent false
105272024-02-15T03:02:01.386Z STATES DS:0 DS:1 DS:2 TOTAL
105282024-02-15T03:02:01.386Z New 0 0 0 0
105292024-02-15T03:02:01.386Z Sent 27 2 2 31
105302024-02-15T03:02:01.386Z Done 2 27 27 56
105312024-02-15T03:02:01.386Z Skipped 0 0 0 0
105322024-02-15T03:02:01.386Z Error 0 0 0 0
105332024-02-15T03:02:01.403ZLast Flush: 1000 1000 1000
105342024-02-15T03:02:01.403ZDownstairs last five completed: 1000
105352024-02-15T03:02:01.403ZUpstairs last five completed: 30 29 28 27 26
105362024-02-15T03:02:01.403Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:01:59.876471114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1511}
105372024-02-15T03:02:01.403ZWrote out file "/tmp/test_repair_verify.data"
105382024-02-15T03:02:01.404Z
105392024-02-15T03:02:01.404ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
105402024-02-15T03:02:01.404ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105412024-02-15T03:02:01.404Z 0 000-019 89 89 89 153 153 153 F T T <---
105422024-02-15T03:02:01.404Z 1 020-039 92 92 92 157 157 157 F F F
105432024-02-15T03:02:01.404Z 2 040-059 92 92 92 157 157 157 F T T <---
105442024-02-15T03:02:01.405Z 3 060-079 86 86 86 148 148 148 F T T <---
105452024-02-15T03:02:01.405Z 4 080-099 92 92 92 158 158 158 F T T <---
105462024-02-15T03:02:01.405Z 5 100-119 92 92 92 157 157 157 T T T
105472024-02-15T03:02:01.405Z 6 120-139 93 93 93 159 159 159 F T T <---
105482024-02-15T03:02:01.405Z 7 140-159 91 91 91 156 156 156 F F F
105492024-02-15T03:02:01.405Z 8 160-179 92 92 92 158 158 158 F F F
105502024-02-15T03:02:01.406Z 9 180-199 89 89 89 153 153 153 F F F
105512024-02-15T03:02:01.406Z 10 200-219 92 92 92 157 157 157 F F F
105522024-02-15T03:02:01.406Z 11 220-239 92 92 92 157 157 157 F F F
105532024-02-15T03:02:01.406Z 12 240-259 92 92 92 157 157 157 F F F
105542024-02-15T03:02:01.406Z 13 260-279 92 92 92 158 158 158 F F F
105552024-02-15T03:02:01.406Z 14 280-299 92 92 92 158 158 158 T T T
105562024-02-15T03:02:01.406Z 15 300-319 89 89 89 153 153 153 F T T <---
105572024-02-15T03:02:01.406Z 16 320-339 89 89 89 153 153 153 F T T <---
105582024-02-15T03:02:01.406Z 17 340-359 92 92 92 158 158 158 F F F
105592024-02-15T03:02:01.406Z 18 360-379 89 89 89 153 153 153 F F F
105602024-02-15T03:02:01.406Z 19 380-399 93 93 93 159 159 159 F T T <---
105612024-02-15T03:02:01.407Z 20 400-419 91 91 91 156 156 156 F T T <---
105622024-02-15T03:02:01.407Z 21 420-439 89 89 89 153 153 153 F T T <---
105632024-02-15T03:02:01.407Z 22 440-459 81 81 81 139 139 139 F F F
105642024-02-15T03:02:01.407Z 23 460-479 92 92 92 158 158 158 F T T <---
105652024-02-15T03:02:01.407Z 24 480-499 86 86 86 148 148 148 F T T <---
105662024-02-15T03:02:01.407Z 25 500-519 91 91 91 156 156 156 F F F
105672024-02-15T03:02:01.408Z 26 520-539 89 89 89 153 153 153 F T T <---
105682024-02-15T03:02:01.408Z 27 540-559 93 93 93 159 159 159 F T T <---
105692024-02-15T03:02:01.408Z 28 560-579 93 93 93 159 159 159 F T T <---
105702024-02-15T03:02:01.408Z 29 580-599 93 93 93 159 159 159 F T T <---
105712024-02-15T03:02:01.408ZMax gen: 93, Max flush: 159
105722024-02-15T03:02:01.409ZError: Difference in extent metadata found!
105732024-02-15T03:02:01.409ZA Difference in extent metadata is expected here
105742024-02-15T03:02:01.409ZOn loop 47
105752024-02-15T03:02:01.409Z
105762024-02-15T03:02:01.409ZVerifying data now
105772024-02-15T03:02:07.325ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
105782024-02-15T03:02:07.371Z287,288c287
105792024-02-15T03:02:07.371Z< 9,
105802024-02-15T03:02:07.372Z< 11,
105812024-02-15T03:02:07.372Z---
105822024-02-15T03:02:07.372Z> 10,
105832024-02-15T03:02:07.372Z289a289
105842024-02-15T03:02:07.373Z> 12,
105852024-02-15T03:02:07.373Zdiff found after verify
105862024-02-15T03:02:07.373ZLoop: 47 Downstairs dump after verify (and repair):
105872024-02-15T03:02:07.373ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105882024-02-15T03:02:07.374Z 0 000-019 94 94 94 160 160 160 F F F
105892024-02-15T03:02:07.374Z 1 020-039 92 92 92 157 157 157 F F F
105902024-02-15T03:02:07.374Z 2 040-059 94 94 94 160 160 160 F F F
105912024-02-15T03:02:07.375Z 3 060-079 94 94 94 160 160 160 F F F
105922024-02-15T03:02:07.375Z 4 080-099 94 94 94 160 160 160 F F F
105932024-02-15T03:02:07.376Z 5 100-119 94 94 94 160 160 160 F F F
105942024-02-15T03:02:07.376Z 6 120-139 94 94 94 160 160 160 F F F
105952024-02-15T03:02:07.376Z 7 140-159 91 91 91 156 156 156 F F F
105962024-02-15T03:02:07.376Z 8 160-179 92 92 92 158 158 158 F F F
105972024-02-15T03:02:07.376Z 9 180-199 89 89 89 153 153 153 F F F
105982024-02-15T03:02:07.376Z 10 200-219 92 92 92 157 157 157 F F F
105992024-02-15T03:02:07.377Z 11 220-239 92 92 92 157 157 157 F F F
106002024-02-15T03:02:07.377Z 12 240-259 92 92 92 157 157 157 F F F
106012024-02-15T03:02:07.378Z 13 260-279 92 92 92 158 158 158 F F F
106022024-02-15T03:02:07.378Z 14 280-299 94 94 94 160 160 160 F F F
106032024-02-15T03:02:07.378Z 15 300-319 94 94 94 160 160 160 F F F
106042024-02-15T03:02:07.378Z 16 320-339 94 94 94 160 160 160 F F F
106052024-02-15T03:02:07.378Z 17 340-359 92 92 92 158 158 158 F F F
106062024-02-15T03:02:07.379Z 18 360-379 89 89 89 153 153 153 F F F
106072024-02-15T03:02:07.379Z 19 380-399 94 94 94 160 160 160 F F F
106082024-02-15T03:02:07.379Z 20 400-419 94 94 94 160 160 160 F F F
106092024-02-15T03:02:07.379Z 21 420-439 94 94 94 160 160 160 F F F
106102024-02-15T03:02:07.379Z 22 440-459 81 81 81 139 139 139 F F F
106112024-02-15T03:02:07.379Z 23 460-479 94 94 94 160 160 160 F F F
106122024-02-15T03:02:07.380Z 24 480-499 94 94 94 160 160 160 F F F
106132024-02-15T03:02:07.380Z 25 500-519 91 91 91 156 156 156 F F F
106142024-02-15T03:02:07.380Z 26 520-539 94 94 94 160 160 160 F F F
106152024-02-15T03:02:07.380Z 27 540-559 94 94 94 160 160 160 F F F
106162024-02-15T03:02:07.381Z 28 560-579 94 94 94 160 160 160 F F F
106172024-02-15T03:02:07.381Z 29 580-599 94 94 94 160 160 160 F F F
106182024-02-15T03:02:07.381ZMax gen: 94, Max flush: 160
106192024-02-15T03:02:07.381Z
106202024-02-15T03:02:07.381ZBegin loop 48
106212024-02-15T03:02:07.381ZDownstairs to restart: 2
106222024-02-15T03:02:07.381Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873365432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522}
106232024-02-15T03:02:07.382Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873641277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522}
106242024-02-15T03:02:07.382Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873659485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522}
106252024-02-15T03:02:07.382Z{"msg":"Crucible ecd58acd-7359-470e-8763-32165ee8ad11 has session id: 6c43f53f-2393-4b20-882d-2bf9e98c8360","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873677382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106262024-02-15T03:02:07.382ZCrucible runtime is spawned
106272024-02-15T03:02:07.382Z{"msg":"Upstairs opts: Upstairs UUID: ecd58acd-7359-470e-8763-32165ee8ad11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873731896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106282024-02-15T03:02:07.382Z{"msg":"Crucible stats registered with UUID: ecd58acd-7359-470e-8763-32165ee8ad11","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873769641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106292024-02-15T03:02:07.383Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873806846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522}
106302024-02-15T03:02:07.383Z{"msg":"ecd58acd-7359-470e-8763-32165ee8ad11 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873918952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106312024-02-15T03:02:07.383Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873950558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106322024-02-15T03:02:07.383Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.873976795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106332024-02-15T03:02:07.384Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874020839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106342024-02-15T03:02:07.384Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874051955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106352024-02-15T03:02:07.384Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874078772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106362024-02-15T03:02:07.384Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874116047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106372024-02-15T03:02:07.384Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874138175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106382024-02-15T03:02:07.384Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874166581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106392024-02-15T03:02:07.384Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874193998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106402024-02-15T03:02:07.384Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874277537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106412024-02-15T03:02:07.385Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874311823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106422024-02-15T03:02:07.385Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:05.874342409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106432024-02-15T03:02:07.385Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:05.874371246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106442024-02-15T03:02:07.385Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:05.87441793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106452024-02-15T03:02:07.386Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874447156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106462024-02-15T03:02:07.386Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874487051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106472024-02-15T03:02:07.386Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.874516777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106482024-02-15T03:02:07.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.875036982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106492024-02-15T03:02:07.387Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.875075047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106502024-02-15T03:02:07.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.875198722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106512024-02-15T03:02:07.387Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.875248266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106522024-02-15T03:02:07.387Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.875634247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106532024-02-15T03:02:07.387Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.875799297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106542024-02-15T03:02:07.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.876163951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106552024-02-15T03:02:07.387Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.876224133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106562024-02-15T03:02:07.387Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.876293145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106572024-02-15T03:02:07.387Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.876345638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106582024-02-15T03:02:07.388Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.876387443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106592024-02-15T03:02:07.388Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:05.87641389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106602024-02-15T03:02:07.970Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.876078594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106612024-02-15T03:02:07.971Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.876153835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106622024-02-15T03:02:07.986Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.877064301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106632024-02-15T03:02:07.986Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.877102396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106642024-02-15T03:02:07.986Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.877680494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106652024-02-15T03:02:07.986Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.87826814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106662024-02-15T03:02:07.986Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878306475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106672024-02-15T03:02:07.987Z{"msg":"[0]R flush_numbers[0..12]: [160, 157, 160, 160, 160, 160, 160, 156, 158, 153, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878326623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106682024-02-15T03:02:07.987Z{"msg":"[0]R generation[0..12]: [94, 92, 94, 94, 94, 94, 94, 91, 92, 89, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878353979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106692024-02-15T03:02:07.987Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878372957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106702024-02-15T03:02:07.987Z{"msg":"[1]R flush_numbers[0..12]: [160, 157, 160, 160, 160, 160, 160, 156, 158, 153, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878391385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106712024-02-15T03:02:07.987Z{"msg":"[1]R generation[0..12]: [94, 92, 94, 94, 94, 94, 94, 91, 92, 89, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878417491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106722024-02-15T03:02:07.987Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878435519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106732024-02-15T03:02:07.987Z{"msg":"[2]R flush_numbers[0..12]: [160, 157, 160, 160, 160, 160, 160, 156, 158, 153, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878462626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106742024-02-15T03:02:07.987Z{"msg":"[2]R generation[0..12]: [94, 92, 94, 94, 94, 94, 94, 91, 92, 89, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878480514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106752024-02-15T03:02:07.987Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.87850664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
106762024-02-15T03:02:07.987Zdownstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106772024-02-15T03:02:07.987Z{"msg":"Max found gen is 95","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878527128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106782024-02-15T03:02:07.987Z{"msg":"Generation requested: 96 >= found:95","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878553574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106792024-02-15T03:02:07.987Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878570192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106802024-02-15T03:02:07.987Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.87858713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106812024-02-15T03:02:07.987Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878612027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106822024-02-15T03:02:07.987Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878627815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106832024-02-15T03:02:07.987Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878645623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106842024-02-15T03:02:07.987Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878672519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106852024-02-15T03:02:07.987Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878700916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106862024-02-15T03:02:07.987Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878719384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106872024-02-15T03:02:07.987Z{"msg":"ecd58acd-7359-470e-8763-32165ee8ad11 is now active with session: 6c43f53f-2393-4b20-882d-2bf9e98c8360","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878736201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106882024-02-15T03:02:07.987Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878752239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522,"session_id":"6c43f53f-2393-4b20-882d-2bf9e98c8360"}
106892024-02-15T03:02:07.988Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:06.878767518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522}
106902024-02-15T03:02:07.988ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106912024-02-15T03:02:07.988ZLoading write count information from file "/tmp/test_repair_verify.data"
106922024-02-15T03:02:07.988ZRun Repair workload
106932024-02-15T03:02:07.988Z01/30 Write block 193 len 28672 data: 11 11 12 11 10 8 7
106942024-02-15T03:02:07.988Z02/30 Read block 293 len 24576
106952024-02-15T03:02:07.988Z03/30 Read block 575 len 4096
106962024-02-15T03:02:07.988Z04/30 Write block 583 len 36864 data: 9 8 8 9 7 6 9 11 13
106972024-02-15T03:02:07.988Z05/30 Write block 434 len 40960 data: 5 7 9 10 10 8 8 9 10 10
106982024-02-15T03:02:07.988Z06/30 Read block 299 len 36864
106992024-02-15T03:02:07.988Z07/30 Read block 64 len 16384
107002024-02-15T03:02:07.989Z08/30 Read block 504 len 24576
107012024-02-15T03:02:07.989Z09/30 Write block 309 len 28672 data: 10 11 11 8 9 8 7
107022024-02-15T03:02:07.989Z10/30 Read block 267 len 20480
107032024-02-15T03:02:07.989Z11/30 Read block 567 len 16384
107042024-02-15T03:02:07.989Z12/30 Read block 27 len 8192
107052024-02-15T03:02:07.989Z13/30 Flush
107062024-02-15T03:02:07.989Z14/30 Write block 573 len 8192 data: 11 12
107072024-02-15T03:02:07.989Z15/30 Write block 553 len 32768 data: 7 8 8 6 10 11 12 11
107082024-02-15T03:02:07.989Z16/30 Read block 459 len 4096
107092024-02-15T03:02:07.989Z17/30 Read block 412 len 24576
107102024-02-15T03:02:07.989Z18/30 Read block 159 len 24576
107112024-02-15T03:02:08.017Z19/30 Read block 338 len 36864
107122024-02-15T03:02:08.017Z20/30 Read block 307 len 24576
107132024-02-15T03:02:08.017Z21/30 Flush
107142024-02-15T03:02:08.017Z22/30 Write block 432 len 8192 data: 6 5
107152024-02-15T03:02:08.017Z23/30 Flush
107162024-02-15T03:02:08.018Z24/30 Write block 247 len 28672 data: 8 7 7 7 7 8 7
107172024-02-15T03:02:08.018Z25/30 Read block 465 len 8192
107182024-02-15T03:02:08.018Z26/30 Write block 146 len 32768 data: 8 9 7 7 6 6 6 8
107192024-02-15T03:02:08.018Z27/30 Read block 138 len 24576
107202024-02-15T03:02:08.018Z28/30 Write block 163 len 28672 data: 10 8 10 11 9 8 7
107212024-02-15T03:02:08.018Z29/30 Write block 257 len 20480 data: 8 8 9 7 7
107222024-02-15T03:02:08.019Z30/30 Write block 179 len 12288 data: 8 8 7
107232024-02-15T03:02:08.019Z----------------------------------------------------------------
107242024-02-15T03:02:08.019Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
107252024-02-15T03:02:08.019ZStates: Active Active Active
107262024-02-15T03:02:08.019ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
107272024-02-15T03:02:08.019Z 1 Acked 1000 Write 7 Done Done Done false
107282024-02-15T03:02:08.020Z 2 Acked 1001 Read 6 Done Done Done false
107292024-02-15T03:02:08.020Z 3 Acked 1002 Read 1 Done Done Sent false
107302024-02-15T03:02:08.020Z 4 Acked 1003 Write 9 Done Done Sent false
107312024-02-15T03:02:08.020Z 5 Acked 1004 Write 10 Done Done Sent false
107322024-02-15T03:02:08.020Z 6 Acked 1005 Read 9 Done Done Sent false
107332024-02-15T03:02:08.020Z 7 Acked 1006 Read 4 Done Done Sent false
107342024-02-15T03:02:08.021Z 8 Acked 1007 Read 6 Done Done Sent false
107352024-02-15T03:02:08.021Z 9 Acked 1008 Write 7 Done Done Sent false
107362024-02-15T03:02:08.021Z 10 Acked 1009 Read 5 Done Done Sent false
107372024-02-15T03:02:08.021Z 11 Acked 1010 Read 4 Done Done Sent false
107382024-02-15T03:02:08.021Z 12 Acked 1011 Read 2 Done Done Sent false
107392024-02-15T03:02:08.021Z 13 Acked 1012 Flush 0 Done Done Sent false
107402024-02-15T03:02:08.022Z 14 Acked 1013 Write 2 Done Done Sent false
107412024-02-15T03:02:08.022Z 15 Acked 1014 Write 8 Done Done Sent false
107422024-02-15T03:02:08.022Z 16 Acked 1015 Read 1 Done Done Sent false
107432024-02-15T03:02:08.022Z 17 Acked 1016 Read 6 Done Done Sent false
107442024-02-15T03:02:08.022Z 18 Acked 1017 Read 6 Done Done Sent false
107452024-02-15T03:02:08.022Z 19 Acked 1018 Read 9 Done Done Sent false
107462024-02-15T03:02:08.023Z 20 Acked 1019 Read 6 Done Done Sent false
107472024-02-15T03:02:08.023Z 21 Acked 1020 Flush 0 Done Done Sent false
107482024-02-15T03:02:08.023Z 22 Acked 1021 Write 2 Done Done Sent false
107492024-02-15T03:02:08.025Z 23 Acked 1022 Flush 0 Done Done Sent false
107502024-02-15T03:02:08.025Z 24 Acked 1023 Write 7 Done Done Sent false
107512024-02-15T03:02:08.025Z 25 Acked 1024 Read 2 Done Done Sent false
107522024-02-15T03:02:08.026Z 26 Acked 1025 Write 8 Done Done Sent false
107532024-02-15T03:02:08.026Z 27 Acked 1026 Read 6 Done Done Sent false
107542024-02-15T03:02:08.026Z 28 Acked 1027 Write 7 Done Done Sent false
107552024-02-15T03:02:08.026Z 29 Acked 1028 Write 5 Sent Sent Sent false
107562024-02-15T03:02:08.026Z 30 Acked 1029 Write 3 Sent Sent Sent false
107572024-02-15T03:02:08.026Z STATES DS:0 DS:1 DS:2 TOTAL
107582024-02-15T03:02:08.026Z New 0 0 0 0
107592024-02-15T03:02:08.026Z Sent 2 2 28 32
107602024-02-15T03:02:08.026Z Done 28 28 2 58
107612024-02-15T03:02:08.026Z Skipped 0 0 0 0
107622024-02-15T03:02:08.027Z Error 0 0 0 0
107632024-02-15T03:02:08.027ZLast Flush: 1022 1022 0
107642024-02-15T03:02:08.027ZDownstairs last five completed:
107652024-02-15T03:02:08.027ZUpstairs last five completed: 30 29 28 27 26
107662024-02-15T03:02:08.027Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:06.911363553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1522}
107672024-02-15T03:02:08.027ZWrote out file "/tmp/test_repair_verify.data"
107682024-02-15T03:02:08.027Z
107692024-02-15T03:02:08.041ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
107702024-02-15T03:02:08.041ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107712024-02-15T03:02:08.042Z 0 000-019 94 94 94 160 160 160 F F F
107722024-02-15T03:02:08.042Z 1 020-039 92 92 92 157 157 157 F F F
107732024-02-15T03:02:08.042Z 2 040-059 94 94 94 160 160 160 F F F
107742024-02-15T03:02:08.042Z 3 060-079 94 94 94 160 160 160 F F F
107752024-02-15T03:02:08.042Z 4 080-099 94 94 94 160 160 160 F F F
107762024-02-15T03:02:08.042Z 5 100-119 94 94 94 160 160 160 F F F
107772024-02-15T03:02:08.042Z 6 120-139 94 94 94 160 160 160 F F F
107782024-02-15T03:02:08.042Z 7 140-159 91 91 91 156 156 156 T T F <---
107792024-02-15T03:02:08.042Z 8 160-179 92 92 92 158 158 158 T T F <---
107802024-02-15T03:02:08.042Z 9 180-199 96 96 89 161 161 153 T T T <---
107812024-02-15T03:02:08.043Z 10 200-219 92 92 92 157 157 157 F F F
107822024-02-15T03:02:08.043Z 11 220-239 92 92 92 157 157 157 F F F
107832024-02-15T03:02:08.043Z 12 240-259 92 92 92 157 157 157 T T F <---
107842024-02-15T03:02:08.043Z 13 260-279 92 92 92 158 158 158 T T F <---
107852024-02-15T03:02:08.043Z 14 280-299 94 94 94 160 160 160 F F F
107862024-02-15T03:02:08.043Z 15 300-319 96 96 94 161 161 160 F F F <---
107872024-02-15T03:02:08.043Z 16 320-339 94 94 94 160 160 160 F F F
107882024-02-15T03:02:08.043Z 17 340-359 92 92 92 158 158 158 F F F
107892024-02-15T03:02:08.043Z 18 360-379 89 89 89 153 153 153 F F F
107902024-02-15T03:02:08.043Z 19 380-399 94 94 94 160 160 160 F F F
107912024-02-15T03:02:08.043Z 20 400-419 94 94 94 160 160 160 F F F
107922024-02-15T03:02:08.043Z 21 420-439 96 96 94 163 163 160 F F F <---
107932024-02-15T03:02:08.043Z 22 440-459 96 96 81 161 161 139 F F F <---
107942024-02-15T03:02:08.044Z 23 460-479 94 94 94 160 160 160 F F F
107952024-02-15T03:02:08.044Z 24 480-499 94 94 94 160 160 160 F F F
107962024-02-15T03:02:08.044Z 25 500-519 91 91 91 156 156 156 F F F
107972024-02-15T03:02:08.044Z 26 520-539 94 94 94 160 160 160 F F F
107982024-02-15T03:02:08.044Z 27 540-559 96 96 94 162 162 160 F F F <---
107992024-02-15T03:02:08.044Z 28 560-579 96 96 94 162 162 160 F F F <---
108002024-02-15T03:02:08.044Z 29 580-599 96 96 94 161 161 160 F F F <---
108012024-02-15T03:02:08.044ZMax gen: 96, Max flush: 163
108022024-02-15T03:02:08.044ZError: Difference in extent metadata found!
108032024-02-15T03:02:08.045ZA Difference in extent metadata is expected here
108042024-02-15T03:02:08.045ZOn loop 48
108052024-02-15T03:02:08.045Z
108062024-02-15T03:02:08.045ZVerifying data now
108072024-02-15T03:02:13.612ZNo change after verify
108082024-02-15T03:02:13.612ZLoop: 48 Downstairs dump after verify (and repair):
108092024-02-15T03:02:13.629ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108102024-02-15T03:02:13.629Z 0 000-019 94 94 94 160 160 160 F F F
108112024-02-15T03:02:13.629Z 1 020-039 92 92 92 157 157 157 F F F
108122024-02-15T03:02:13.629Z 2 040-059 94 94 94 160 160 160 F F F
108132024-02-15T03:02:13.630Z 3 060-079 94 94 94 160 160 160 F F F
108142024-02-15T03:02:13.630Z 4 080-099 94 94 94 160 160 160 F F F
108152024-02-15T03:02:13.630Z 5 100-119 94 94 94 160 160 160 F F F
108162024-02-15T03:02:13.630Z 6 120-139 94 94 94 160 160 160 F F F
108172024-02-15T03:02:13.630Z 7 140-159 97 97 97 164 164 164 F F F
108182024-02-15T03:02:13.630Z 8 160-179 97 97 97 164 164 164 F F F
108192024-02-15T03:02:13.630Z 9 180-199 97 97 97 164 164 164 F F F
108202024-02-15T03:02:13.630Z 10 200-219 92 92 92 157 157 157 F F F
108212024-02-15T03:02:13.631Z 11 220-239 92 92 92 157 157 157 F F F
108222024-02-15T03:02:13.631Z 12 240-259 97 97 97 164 164 164 F F F
108232024-02-15T03:02:13.631Z 13 260-279 97 97 97 164 164 164 F F F
108242024-02-15T03:02:13.631Z 14 280-299 94 94 94 160 160 160 F F F
108252024-02-15T03:02:13.631Z 15 300-319 96 96 96 161 161 161 F F F
108262024-02-15T03:02:13.631Z 16 320-339 94 94 94 160 160 160 F F F
108272024-02-15T03:02:13.631Z 17 340-359 92 92 92 158 158 158 F F F
108282024-02-15T03:02:13.631Z 18 360-379 89 89 89 153 153 153 F F F
108292024-02-15T03:02:13.631Z 19 380-399 94 94 94 160 160 160 F F F
108302024-02-15T03:02:13.632Z 20 400-419 94 94 94 160 160 160 F F F
108312024-02-15T03:02:13.632Z 21 420-439 96 96 96 163 163 163 F F F
108322024-02-15T03:02:13.632Z 22 440-459 96 96 96 161 161 161 F F F
108332024-02-15T03:02:13.632Z 23 460-479 94 94 94 160 160 160 F F F
108342024-02-15T03:02:13.632Z 24 480-499 94 94 94 160 160 160 F F F
108352024-02-15T03:02:13.632Z 25 500-519 91 91 91 156 156 156 F F F
108362024-02-15T03:02:13.632Z 26 520-539 94 94 94 160 160 160 F F F
108372024-02-15T03:02:13.633Z 27 540-559 96 96 96 162 162 162 F F F
108382024-02-15T03:02:13.633Z 28 560-579 96 96 96 162 162 162 F F F
108392024-02-15T03:02:13.633Z 29 580-599 96 96 96 161 161 161 F F F
108402024-02-15T03:02:13.633ZMax gen: 97, Max flush: 164
108412024-02-15T03:02:13.638Z
108422024-02-15T03:02:13.638ZBegin loop 49
108432024-02-15T03:02:13.639ZDownstairs to restart: 1
108442024-02-15T03:02:13.658Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560168815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530}
108452024-02-15T03:02:13.659Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"Crucible runtime is spawned
108462024-02-15T03:02:13.659Z56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560547108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530}
108472024-02-15T03:02:13.659Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560594832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530}
108482024-02-15T03:02:13.659Z{"msg":"Crucible 9777d60f-76ea-4512-882b-5f4a33fdef1f has session id: 79f48360-b3a4-4ae1-a97d-e99f77147f5f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560614999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108492024-02-15T03:02:13.659Z{"msg":"Upstairs opts: Upstairs UUID: 9777d60f-76ea-4512-882b-5f4a33fdef1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560652955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108502024-02-15T03:02:13.659Z{"msg":"Crucible stats registered with UUID: 9777d60f-76ea-4512-882b-5f4a33fdef1f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560668973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108512024-02-15T03:02:13.659Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.5606908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530}
108522024-02-15T03:02:13.659Z{"msg":"9777d60f-76ea-4512-882b-5f4a33fdef1f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560715447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108532024-02-15T03:02:13.659Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560741844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108542024-02-15T03:02:13.659Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560777869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108552024-02-15T03:02:13.659Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560816104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108562024-02-15T03:02:13.660Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560841661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108572024-02-15T03:02:13.660Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.560954827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108582024-02-15T03:02:13.660Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561000251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108592024-02-15T03:02:13.660Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561030297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108602024-02-15T03:02:13.660Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561064313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108612024-02-15T03:02:13.660Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.56109324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108622024-02-15T03:02:13.660Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561123576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108632024-02-15T03:02:13.660Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:12.561156361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108642024-02-15T03:02:13.660Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:12.561188848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108652024-02-15T03:02:13.660Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561232072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108662024-02-15T03:02:13.660Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:12.561281376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108672024-02-15T03:02:13.660Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561319501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108682024-02-15T03:02:13.660Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561341698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108692024-02-15T03:02:13.660Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561475112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108702024-02-15T03:02:13.660Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561925135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108712024-02-15T03:02:13.661Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561962411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108722024-02-15T03:02:13.661Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561979178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108732024-02-15T03:02:13.661Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.561992997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108742024-02-15T03:02:13.661Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.562561065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108752024-02-15T03:02:13.661Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.562702518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108762024-02-15T03:02:13.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.563108317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108772024-02-15T03:02:13.661Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.563167589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108782024-02-15T03:02:13.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.563204005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108792024-02-15T03:02:13.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.563228012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108802024-02-15T03:02:13.661Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.563256668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108812024-02-15T03:02:13.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:12.563281075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108822024-02-15T03:02:14.654Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.563174591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108832024-02-15T03:02:14.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.563334741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108842024-02-15T03:02:14.662Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.564288552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108852024-02-15T03:02:14.663Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.564351034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108862024-02-15T03:02:14.663Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.564877918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108872024-02-15T03:02:14.663Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565429799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108882024-02-15T03:02:14.663Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565472834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108892024-02-15T03:02:14.663Z{"msg":"[0]R flush_numbers[0..12]: [160, 157, 160, 160, 160, 160, 160, 164, 164, 164, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565493501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108902024-02-15T03:02:14.664Z{"msg":"[0]R generation[0..12]: [94, 92, 94, 94, 94, 94, 94, 97, 97, 97, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565521657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108912024-02-15T03:02:14.664Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565541065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108922024-02-15T03:02:14.664Z{"msg":"[1]R flush_numbers[0..12]: [160, 157, 160, 160, 160, 160, 160, 164, 164, 164, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565556653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108932024-02-15T03:02:14.664Z{"msg":"[1]R generation[0..12]: [94, 92, 94, 94, 94, 94, 94, 97, 97, 97, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.56557774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108942024-02-15T03:02:14.665Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565592688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108952024-02-15T03:02:14.665Z{"msg":"[2]R flush_numbers[0..12]: [160, 157, 160, 160, 160, 160, 160, 164, 164, 164, 157, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565615436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108962024-02-15T03:02:14.665Z{"msg":"[2]R generation[0..12]: [94, 92, 94, 94, 94, 94, 94, 97, 97, 97, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565629784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108972024-02-15T03:02:14.666Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
108982024-02-15T03:02:14.666Z,"time":"2024-02-15T03:02:13.565643792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
108992024-02-15T03:02:14.666Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.5656608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109002024-02-15T03:02:14.666Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565682037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109012024-02-15T03:02:14.666Z{"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565695575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109022024-02-15T03:02:14.666Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565712153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109032024-02-15T03:02:14.666Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.56573738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109042024-02-15T03:02:14.666Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565760118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109052024-02-15T03:02:14.666Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565793373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109062024-02-15T03:02:14.666Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565827419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109072024-02-15T03:02:14.666Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565853806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109082024-02-15T03:02:14.666Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565886222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109092024-02-15T03:02:14.666Z{"msg":"9777d60f-76ea-4512-882b-5f4a33fdef1f is now active with session: 79f48360-b3a4-4ae1-a97d-e99f77147f5f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565911248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109102024-02-15T03:02:14.666Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
109112024-02-15T03:02:14.666Z,"time":"2024-02-15T03:02:13.565943175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530,"session_id":"79f48360-b3a4-4ae1-a97d-e99f77147f5f"}
109122024-02-15T03:02:14.667Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:13.565969231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530}
109132024-02-15T03:02:14.667ZLoading write count information from file "/tmp/test_repair_verify.data"
109142024-02-15T03:02:14.667ZRun Repair workload
109152024-02-15T03:02:14.667Z01/30 Write block 84 len 12288 data: 7 7 7
109162024-02-15T03:02:14.667Z02/30 Read block 330 len 4096
109172024-02-15T03:02:14.667Z03/30 Write block 39 len 12288 data: 5 4 5
109182024-02-15T03:02:14.667Z04/30 Write block 86 len 24576 data: 8 7 5 4 5 4
109192024-02-15T03:02:14.667Z05/30 Read block 399 len 32768
109202024-02-15T03:02:14.668Z06/30 Write block 70 len 16384 data: 5 5 4 3
109212024-02-15T03:02:14.668Z07/30 Read block 193 len 28672
109222024-02-15T03:02:14.668Z08/30 Write block 166 len 16384 data: 12 10 9 8
109232024-02-15T03:02:14.668Z09/30 Write block 340 len 32768 data: 7 7 7 5 6 7 8 7
109242024-02-15T03:02:14.668Z10/30 Read block 121 len 24576
109252024-02-15T03:02:14.668Z11/30 Write block 204 len 8192 data: 7 6
109262024-02-15T03:02:14.673Z12/30 Write block 150 len 36864 data: 7 7 7 9 8 8 12 13 12
109272024-02-15T03:02:14.673Z13/30 Read block 468 len 32768
109282024-02-15T03:02:14.673Z14/30 Write block 308 len 24576 data: 9 11 12 12 9 10
109292024-02-15T03:02:14.673Z15/30 Read block 23 len 12288
109302024-02-15T03:02:14.674Z16/30 Flush
109312024-02-15T03:02:14.674Z17/30 Write block 528 len 12288 data: 7 7 7
109322024-02-15T03:02:14.674Z18/30 Write block 478 len 20480 data: 10 12 10 10 9
109332024-02-15T03:02:14.674Z19/30 Read block 340 len 20480
109342024-02-15T03:02:14.674Z20/30 Write block 344 len 20480 data: 7 8 9 8 7
109352024-02-15T03:02:14.675Z21/30 Write block 290 len 32768 data: 15 13 12 12 12 10 9 7
109362024-02-15T03:02:14.675Z22/30 Read block 173 len 32768
109372024-02-15T03:02:14.693Z23/30 Read block 410 len 16384
109382024-02-15T03:02:14.693Z24/30 Read block 216 len 16384
109392024-02-15T03:02:14.693Z25/30 Write block 221 len 20480 data: 11 10 8 8 9
109402024-02-15T03:02:14.693Z26/30 Flush
109412024-02-15T03:02:14.693Z27/30 Write block 314 len 40960 data: 9 8 8 8 8 7 6 6 6 8
109422024-02-15T03:02:14.694Z28/30 Read block 299 len 32768
109432024-02-15T03:02:14.694Z29/30 Write block 138 len 40960 data: 8 7 9 10 9 8 9 9 9 10
109442024-02-15T03:02:14.694Z30/30 Write block 492 len 4096 data: 8
109452024-02-15T03:02:14.694Z----------------------------------------------------------------
109462024-02-15T03:02:14.694Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
109472024-02-15T03:02:14.694ZStates: Active Active Active
109482024-02-15T03:02:14.695ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109492024-02-15T03:02:14.695Z 1 Acked 1000 Write 3 Done Done Done false
109502024-02-15T03:02:14.695Z 2 Acked 1001 Read 1 Done Done Done false
109512024-02-15T03:02:14.695Z 3 Acked 1002 Write 3 Done Sent Done false
109522024-02-15T03:02:14.695Z 4 Acked 1003 Write 6 Done Sent Done false
109532024-02-15T03:02:14.695Z 5 Acked 1004 Read 8 Done Sent Done false
109542024-02-15T03:02:14.695Z 6 Acked 1005 Write 4 Done Sent Done false
109552024-02-15T03:02:14.696Z 7 Acked 1006 Read 7 Done Sent Done false
109562024-02-15T03:02:14.696Z 8 Acked 1007 Write 4 Done Sent Done false
109572024-02-15T03:02:14.696Z 9 Acked 1008 Write 8 Done Sent Done false
109582024-02-15T03:02:14.696Z 10 Acked 1009 Read 6 Done Sent Done false
109592024-02-15T03:02:14.696Z 11 Acked 1010 Write 2 Done Sent Done false
109602024-02-15T03:02:14.696Z 12 Acked 1011 Write 9 Done Sent Done false
109612024-02-15T03:02:14.696Z 13 Acked 1012 Read 8 Done Sent Done false
109622024-02-15T03:02:14.696Z 14 Acked 1013 Write 6 Done Sent Done false
109632024-02-15T03:02:14.696Z 15 Acked 1014 Read 3 Done Sent Done false
109642024-02-15T03:02:14.697Z 16 Acked 1015 Flush 0 Done Sent Done false
109652024-02-15T03:02:14.697Z 17 Acked 1016 Write 3 Done Sent Done false
109662024-02-15T03:02:14.697Z 18 Acked 1017 Write 5 Done Sent Done false
109672024-02-15T03:02:14.697Z 19 Acked 1018 Read 5 Done Sent Done false
109682024-02-15T03:02:14.697Z 20 Acked 1019 Write 5 Done Sent Done false
109692024-02-15T03:02:14.697Z 21 Acked 1020 Write 8 Done Sent Done false
109702024-02-15T03:02:14.697Z 22 Acked 1021 Read 8 Done Sent Done false
109712024-02-15T03:02:14.698Z 23 Acked 1022 Read 4 Done Sent Done false
109722024-02-15T03:02:14.698Z 24 Acked 1023 Read 4 Done Sent Done false
109732024-02-15T03:02:14.698Z 25 Acked 1024 Write 5 Done Sent Done false
109742024-02-15T03:02:14.698Z 26 Acked 1025 Flush 0 Done Sent Done false
109752024-02-15T03:02:14.698Z 27 Acked 1026 Write 10 Done Sent Done false
109762024-02-15T03:02:14.698Z 28 Acked 1027 Read 8 Done Sent Done false
109772024-02-15T03:02:14.698Z 29 Acked 1028 Write 10 Sent Sent Sent false
109782024-02-15T03:02:14.698Z 30 Acked 1029 Write 1 Sent Sent Sent false
109792024-02-15T03:02:14.698Z STATES DS:0 DS:1 DS:2 TOTAL
109802024-02-15T03:02:14.698Z New 0 0 0 0
109812024-02-15T03:02:14.699Z Sent 2 28 2 32
109822024-02-15T03:02:14.699Z Done 28 2 28 58
109832024-02-15T03:02:14.699Z Skipped 0 0 0 0
109842024-02-15T03:02:14.699Z Error 0 0 0 0
109852024-02-15T03:02:14.700ZLast Flush: 1025 0 1025
109862024-02-15T03:02:14.700ZDownstairs last five completed:
109872024-02-15T03:02:14.700ZUpstairs last five completed: 30 29 28 27 26
109882024-02-15T03:02:14.700Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:13.599226974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1530}
109892024-02-15T03:02:14.700ZWrote out file "/tmp/test_repair_verify.data"
109902024-02-15T03:02:14.700Z
109912024-02-15T03:02:14.721ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109922024-02-15T03:02:14.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109932024-02-15T03:02:14.744Z 0 000-019 94 94 94 160 160 160 F F F
109942024-02-15T03:02:14.744Z 1 020-039 98 92 98 165 157 165 F F F <---
109952024-02-15T03:02:14.744Z 2 040-059 98 94 98 165 160 165 F F F <---
109962024-02-15T03:02:14.744Z 3 060-079 98 94 98 165 160 165 F F F <---
109972024-02-15T03:02:14.744Z 4 080-099 98 94 98 165 160 165 F T F <---
109982024-02-15T03:02:14.744Z 5 100-119 94 94 94 160 160 160 F F F
109992024-02-15T03:02:14.744Z 6 120-139 94 94 94 160 160 160 T F T <---
110002024-02-15T03:02:14.744Z 7 140-159 98 97 98 165 164 165 T F T <---
110012024-02-15T03:02:14.744Z 8 160-179 98 97 98 165 164 165 F F F <---
110022024-02-15T03:02:14.745Z 9 180-199 97 97 97 164 164 164 F F F
110032024-02-15T03:02:14.745Z 10 200-219 98 92 98 165 157 165 F F F <---
110042024-02-15T03:02:14.745Z 11 220-239 98 92 98 166 157 166 F F F <---
110052024-02-15T03:02:14.745Z 12 240-259 97 97 97 164 164 164 F F F
110062024-02-15T03:02:14.745Z 13 260-279 97 97 97 164 164 164 F F F
110072024-02-15T03:02:14.745Z 14 280-299 98 94 98 166 160 166 F F F <---
110082024-02-15T03:02:14.745Z 15 300-319 98 96 98 165 161 165 T F T <---
110092024-02-15T03:02:14.745Z 16 320-339 94 94 94 160 160 160 T F T <---
110102024-02-15T03:02:14.746Z 17 340-359 98 92 98 166 158 166 F F F <---
110112024-02-15T03:02:14.746Z 18 360-379 89 89 89 153 153 153 F F F
110122024-02-15T03:02:14.746Z 19 380-399 94 94 94 160 160 160 F F F
110132024-02-15T03:02:14.746Z 20 400-419 94 94 94 160 160 160 F F F
110142024-02-15T03:02:14.746Z 21 420-439 96 96 96 163 163 163 F F F
110152024-02-15T03:02:14.746Z 22 440-459 96 96 96 161 161 161 F F F
110162024-02-15T03:02:14.746Z 23 460-479 98 94 98 166 160 166 F F F <---
110172024-02-15T03:02:14.746Z 24 480-499 98 94 98 166 160 166 T F T <---
110182024-02-15T03:02:14.746Z 25 500-519 91 91 91 156 156 156 F F F
110192024-02-15T03:02:14.747Z 26 520-539 98 94 98 166 160 166 F F F <---
110202024-02-15T03:02:14.747Z 27 540-559 96 96 96 162 162 162 F F F
110212024-02-15T03:02:14.747Z 28 560-579 96 96 96 162 162 162 F F F
110222024-02-15T03:02:14.747Z 29 580-599 96 96 96 161 161 161 F F F
110232024-02-15T03:02:14.747ZMax gen: 98, Max flush: 166
110242024-02-15T03:02:14.747ZError: Difference in extent metadata found!
110252024-02-15T03:02:14.747ZA Difference in extent metadata is expected here
110262024-02-15T03:02:14.747ZOn loop 49
110272024-02-15T03:02:14.747Z
110282024-02-15T03:02:14.747ZVerifying data now
110292024-02-15T03:02:20.520ZNo change after verify
110302024-02-15T03:02:20.520ZLoop: 49 Downstairs dump after verify (and repair):
110312024-02-15T03:02:20.537ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110322024-02-15T03:02:20.537Z 0 000-019 94 94 94 160 160 160 F F F
110332024-02-15T03:02:20.538Z 1 020-039 98 98 98 165 165 165 F F F
110342024-02-15T03:02:20.538Z 2 040-059 98 98 98 165 165 165 F F F
110352024-02-15T03:02:20.538Z 3 060-079 98 98 98 165 165 165 F F F
110362024-02-15T03:02:20.538Z 4 080-099 98 98 98 165 165 165 F F F
110372024-02-15T03:02:20.538Z 5 100-119 94 94 94 160 160 160 F F F
110382024-02-15T03:02:20.538Z 6 120-139 99 99 99 167 167 167 F F F
110392024-02-15T03:02:20.538Z 7 140-159 99 99 99 167 167 167 F F F
110402024-02-15T03:02:20.539Z 8 160-179 98 98 98 165 165 165 F F F
110412024-02-15T03:02:20.539Z 9 180-199 97 97 97 164 164 164 F F F
110422024-02-15T03:02:20.539Z 10 200-219 98 98 98 165 165 165 F F F
110432024-02-15T03:02:20.552Z 11 220-239 98 98 98 166 166 166 F F F
110442024-02-15T03:02:20.552Z 12 240-259 97 97 97 164 164 164 F F F
110452024-02-15T03:02:20.552Z 13 260-279 97 97 97 164 164 164 F F F
110462024-02-15T03:02:20.552Z 14 280-299 98 98 98 166 166 166 F F F
110472024-02-15T03:02:20.552Z 15 300-319 99 99 99 167 167 167 F F F
110482024-02-15T03:02:20.552Z 16 320-339 99 99 99 167 167 167 F F F
110492024-02-15T03:02:20.552Z 17 340-359 98 98 98 166 166 166 F F F
110502024-02-15T03:02:20.552Z 18 360-379 89 89 89 153 153 153 F F F
110512024-02-15T03:02:20.552Z 19 380-399 94 94 94 160 160 160 F F F
110522024-02-15T03:02:20.553Z 20 400-419 94 94 94 160 160 160 F F F
110532024-02-15T03:02:20.553Z 21 420-439 96 96 96 163 163 163 F F F
110542024-02-15T03:02:20.553Z 22 440-459 96 96 96 161 161 161 F F F
110552024-02-15T03:02:20.553Z 23 460-479 98 98 98 166 166 166 F F F
110562024-02-15T03:02:20.553Z 24 480-499 99 99 99 167 167 167 F F F
110572024-02-15T03:02:20.554Z 25 500-519 91 91 91 156 156 156 F F F
110582024-02-15T03:02:20.554Z 26 520-539 98 98 98 166 166 166 F F F
110592024-02-15T03:02:20.554Z 27 540-559 96 96 96 162 162 162 F F F
110602024-02-15T03:02:20.554Z 28 560-579 96 96 96 162 162 162 F F F
110612024-02-15T03:02:20.554Z 29 580-599 96 96 96 161 161 161 F F F
110622024-02-15T03:02:20.555ZMax gen: 99, Max flush: 167
110632024-02-15T03:02:20.555Z
110642024-02-15T03:02:20.555ZBegin loop 50
110652024-02-15T03:02:20.556ZDownstairs to restart: 2
110662024-02-15T03:02:20.586Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.47322961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538}
110672024-02-15T03:02:20.586Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"Crucible runtime is spawned
110682024-02-15T03:02:20.587Zx86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473532052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538}
110692024-02-15T03:02:20.587Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473572727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538}
110702024-02-15T03:02:20.587Z{"msg":"Crucible dd1deffc-cdcb-400a-b5c9-595eb7cd545d has session id: 520dcbd7-27cf-4a70-bd73-51dc05fc985e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473607342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110712024-02-15T03:02:20.587Z{"msg":"Upstairs opts: Upstairs UUID: dd1deffc-cdcb-400a-b5c9-595eb7cd545d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473665025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110722024-02-15T03:02:20.587Z{"msg":"Crucible stats registered with UUID: dd1deffc-cdcb-400a-b5c9-595eb7cd545d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473687902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110732024-02-15T03:02:20.587Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473711169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538}
110742024-02-15T03:02:20.587Z{"msg":"dd1deffc-cdcb-400a-b5c9-595eb7cd545d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473778101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110752024-02-15T03:02:20.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473814016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110762024-02-15T03:02:20.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473866979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110772024-02-15T03:02:20.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473899966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110782024-02-15T03:02:20.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473933141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110792024-02-15T03:02:20.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.473963087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110802024-02-15T03:02:20.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474006382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110812024-02-15T03:02:20.589Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474034618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110822024-02-15T03:02:20.589Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474089972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110832024-02-15T03:02:20.589Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474127737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110842024-02-15T03:02:20.589Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:19.474191439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110852024-02-15T03:02:20.589Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:19.474230614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110862024-02-15T03:02:20.590Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:19.474277508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110872024-02-15T03:02:20.590Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474328592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110882024-02-15T03:02:20.590Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474384525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110892024-02-15T03:02:20.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474432959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110902024-02-15T03:02:20.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474570171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110912024-02-15T03:02:20.590Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.474699575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110922024-02-15T03:02:20.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.475315178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110932024-02-15T03:02:20.591Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.475343234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110942024-02-15T03:02:20.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.475361152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110952024-02-15T03:02:20.591Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.475399218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110962024-02-15T03:02:20.591Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.47593724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110972024-02-15T03:02:20.591Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.476072573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110982024-02-15T03:02:20.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.476456495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
110992024-02-15T03:02:20.591Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.476502829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111002024-02-15T03:02:20.592Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.476521137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111012024-02-15T03:02:20.592Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.476544404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111022024-02-15T03:02:20.592Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.476560292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111032024-02-15T03:02:20.592Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:19.47657437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111042024-02-15T03:02:21.562Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.476246894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111052024-02-15T03:02:21.562Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.476323415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111062024-02-15T03:02:21.575Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.477144462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111072024-02-15T03:02:21.575Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.477184437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111082024-02-15T03:02:21.575Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.477653368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111092024-02-15T03:02:21.575Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478241434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111102024-02-15T03:02:21.575Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478292008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111112024-02-15T03:02:21.575Z{"msg":"[0]R flush_numbers[0..12]: [160, 165, 165, 165, 165, 160, 167, 167, 165, 164, 165, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478330133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111122024-02-15T03:02:21.575Z{"msg":"[0]R generation[0..12]: [94, 98, 98, 98, 98, 94, 99, 99, 98, 97, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478373438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111132024-02-15T03:02:21.575Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478393075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111142024-02-15T03:02:21.575Z{"msg":"[1]R flush_numbers[0..12]: [160, 165, 165, 165, 165, 160, 167, 167, 165, 164, 165, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478421942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111152024-02-15T03:02:21.575Z{"msg":"[1]R generation[0..12]: [94, 98, 98, 98, 98, 94, 99, 99, 98, 97, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478449188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111162024-02-15T03:02:21.575ZWait for a query_work_queue command to finish before sending IO
111172024-02-15T03:02:21.575Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478500752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111182024-02-15T03:02:21.575Z{"msg":"[2]R flush_numbers[0..12]: [160, 165, 165, 165, 165, 160, 167, 167, 165, 164, 165, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478547496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111192024-02-15T03:02:21.575Z{"msg":"[2]R generation[0..12]: [94, 98, 98, 98, 98, 94, 99, 99, 98, 97, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478579502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111202024-02-15T03:02:21.575Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478598589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111212024-02-15T03:02:21.575Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478629376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111222024-02-15T03:02:21.575Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.47867501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111232024-02-15T03:02:21.575Z{"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478715635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111242024-02-15T03:02:21.575Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478735652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111252024-02-15T03:02:21.576Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478761739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111262024-02-15T03:02:21.576Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478797265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111272024-02-15T03:02:21.576Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478828161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111282024-02-15T03:02:21.576Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478843509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111292024-02-15T03:02:21.576Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478869426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111302024-02-15T03:02:21.576Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478906671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111312024-02-15T03:02:21.576Z{"msg":"dd1deffc-cdcb-400a-b5c9-595eb7cd545d is now active with session: 520dcbd7-27cf-4a70-bd73-51dc05fc985e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478934947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111322024-02-15T03:02:21.576Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478949506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538,"session_id":"520dcbd7-27cf-4a70-bd73-51dc05fc985e"}
111332024-02-15T03:02:21.576Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:20.478982201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538}
111342024-02-15T03:02:21.576ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111352024-02-15T03:02:21.576ZLoading write count information from file "/tmp/test_repair_verify.data"
111362024-02-15T03:02:21.576ZRun Repair workload
111372024-02-15T03:02:21.576Z01/30 Write block 159 len 28672 data: 9 7 8 12 11 9 11
111382024-02-15T03:02:21.576Z02/30 Read block 559 len 36864
111392024-02-15T03:02:21.582Z03/30 Flush
111402024-02-15T03:02:21.583Z04/30 Write block 231 len 36864 data: 9 10 10 11 10 10 8 7 7
111412024-02-15T03:02:21.583Z05/30 Read block 433 len 40960
111422024-02-15T03:02:21.583Z06/30 Read block 414 len 12288
111432024-02-15T03:02:21.583Z07/30 Read block 361 len 16384
111442024-02-15T03:02:21.583Z08/30 Read block 255 len 40960
111452024-02-15T03:02:21.583Z09/30 Write block 35 len 20480 data: 8 7 7 8 6
111462024-02-15T03:02:21.583Z10/30 Read block 289 len 24576
111472024-02-15T03:02:21.583Z11/30 Read block 18 len 16384
111482024-02-15T03:02:21.584Z12/30 Read block 430 len 4096
111492024-02-15T03:02:21.584Z13/30 Read block 273 len 36864
111502024-02-15T03:02:21.584Z14/30 Read block 139 len 36864
111512024-02-15T03:02:21.584Z15/30 Read block 73 len 12288
111522024-02-15T03:02:21.590Z16/30 Write block 538 len 28672 data: 8 9 7 8 10 10 5
111532024-02-15T03:02:21.591Z17/30 Read block 534 len 8192
111542024-02-15T03:02:21.591Z18/30 Write block 541 len 12288 data: 9 11 11
111552024-02-15T03:02:21.591Z19/30 Read block 406 len 36864
111562024-02-15T03:02:21.591Z20/30 Read block 183 len 28672
111572024-02-15T03:02:21.591Z21/30 Read block 533 len 28672
111582024-02-15T03:02:21.592Z22/30 Read block 405 len 12288
111592024-02-15T03:02:21.592Z23/30 Write block 461 len 28672 data: 8 7 5 5 6 8 8
111602024-02-15T03:02:21.611Z24/30 Write block 87 len 32768 data: 8 6 5 6 5 4 4 7
111612024-02-15T03:02:21.612Z25/30 Write block 283 len 12288 data: 8 10 12
111622024-02-15T03:02:21.612Z26/30 Read block 240 len 12288
111632024-02-15T03:02:21.612Z27/30 Read block 594 len 4096
111642024-02-15T03:02:21.612Z28/30 Write block 234 len 36864 data: 12 11 11 9 8 8 8 9 9
111652024-02-15T03:02:21.612Z29/30 Read block 4 len 4096
111662024-02-15T03:02:21.612Z30/30 Write block 498 len 4096 data: 10
111672024-02-15T03:02:21.612Z----------------------------------------------------------------
111682024-02-15T03:02:21.612Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
111692024-02-15T03:02:21.612ZStates: Active Active Active
111702024-02-15T03:02:21.612ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
111712024-02-15T03:02:21.613Z 1 Acked 1000 Write 7 Done Done Done false
111722024-02-15T03:02:21.613Z 2 Acked 1001 Read 9 Done Done Done false
111732024-02-15T03:02:21.613Z 3 Acked 1002 Flush 0 Done Done Sent false
111742024-02-15T03:02:21.613Z 4 Acked 1003 Write 9 Done Done Sent false
111752024-02-15T03:02:21.613Z 5 Acked 1004 Read 10 Done Done Sent false
111762024-02-15T03:02:21.613Z 6 Acked 1005 Read 3 Done Done Sent false
111772024-02-15T03:02:21.613Z 7 Acked 1006 Read 4 Done Done Sent false
111782024-02-15T03:02:21.613Z 8 Acked 1007 Read 10 Done Done Sent false
111792024-02-15T03:02:21.614Z 9 Acked 1008 Write 5 Done Done Sent false
111802024-02-15T03:02:21.614Z 10 Acked 1009 Read 6 Done Done Sent false
111812024-02-15T03:02:21.614Z 11 Acked 1010 Read 4 Done Done Sent false
111822024-02-15T03:02:21.614Z 12 Acked 1011 Read 1 Done Done Sent false
111832024-02-15T03:02:21.614Z 13 Acked 1012 Read 9 Done Done Sent false
111842024-02-15T03:02:21.614Z 14 Acked 1013 Read 9 Done Done Sent false
111852024-02-15T03:02:21.614Z 15 Acked 1014 Read 3 Done Done Sent false
111862024-02-15T03:02:21.614Z 16 Acked 1015 Write 7 Done Done Sent false
111872024-02-15T03:02:21.614Z 17 Acked 1016 Read 2 Done Done Sent false
111882024-02-15T03:02:21.614Z 18 Acked 1017 Write 3 Done Done Sent false
111892024-02-15T03:02:21.615Z 19 Acked 1018 Read 9 Done Done Sent false
111902024-02-15T03:02:21.615Z 20 Acked 1019 Read 7 Done Done Sent false
111912024-02-15T03:02:21.615Z 21 Acked 1020 Read 7 Done Done Sent false
111922024-02-15T03:02:21.615Z 22 Acked 1021 Read 3 Done Done Sent false
111932024-02-15T03:02:21.615Z 23 Acked 1022 Write 7 Done Done Sent false
111942024-02-15T03:02:21.615Z 24 Acked 1023 Write 8 Done Done Sent false
111952024-02-15T03:02:21.615Z 25 Acked 1024 Write 3 Done Done Sent false
111962024-02-15T03:02:21.615Z 26 Acked 1025 Read 3 Done Done Sent false
111972024-02-15T03:02:21.615Z 27 Acked 1026 Read 1 Done Done Sent false
111982024-02-15T03:02:21.615Z 28 Acked 1027 Write 9 Done Done Sent false
111992024-02-15T03:02:21.616Z 29 Acked 1028 Read 1 Done Done Sent false
112002024-02-15T03:02:21.616Z 30 Acked 1029 Write 1 Sent Sent Sent false
112012024-02-15T03:02:21.616Z STATES DS:0 DS:1 DS:2 TOTAL
112022024-02-15T03:02:21.616Z New 0 0 0 0
112032024-02-15T03:02:21.616Z Sent 1 1 28 30
112042024-02-15T03:02:21.616Z Done 29 29 2 60
112052024-02-15T03:02:21.617Z Skipped 0 0 0 0
112062024-02-15T03:02:21.617Z Error 0 0 0 0
112072024-02-15T03:02:21.617ZLast Flush: 1002 1002 0
112082024-02-15T03:02:21.617ZDownstairs last five completed:
112092024-02-15T03:02:21.617ZUpstairs last five completed: 30 29 28 27 26
112102024-02-15T03:02:21.617Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:20.50939896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1538}
112112024-02-15T03:02:21.617ZWrote out file "/tmp/test_repair_verify.data"
112122024-02-15T03:02:21.633Z
112132024-02-15T03:02:21.633ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112142024-02-15T03:02:21.633ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112152024-02-15T03:02:21.634Z 0 000-019 94 94 94 160 160 160 F F F
112162024-02-15T03:02:21.634Z 1 020-039 98 98 98 165 165 165 T T F <---
112172024-02-15T03:02:21.634Z 2 040-059 98 98 98 165 165 165 F F F
112182024-02-15T03:02:21.635Z 3 060-079 98 98 98 165 165 165 F F F
112192024-02-15T03:02:21.635Z 4 080-099 98 98 98 165 165 165 T T F <---
112202024-02-15T03:02:21.635Z 5 100-119 94 94 94 160 160 160 F F F
112212024-02-15T03:02:21.635Z 6 120-139 99 99 99 167 167 167 F F F
112222024-02-15T03:02:21.635Z 7 140-159 100 100 99 168 168 167 F F T <---
112232024-02-15T03:02:21.635Z 8 160-179 100 100 98 168 168 165 F F T <---
112242024-02-15T03:02:21.635Z 9 180-199 97 97 97 164 164 164 F F F
112252024-02-15T03:02:21.635Z 10 200-219 98 98 98 165 165 165 F F F
112262024-02-15T03:02:21.636Z 11 220-239 98 98 98 166 166 166 T T F <---
112272024-02-15T03:02:21.636Z 12 240-259 97 97 97 164 164 164 T T F <---
112282024-02-15T03:02:21.636Z 13 260-279 97 97 97 164 164 164 F F F
112292024-02-15T03:02:21.637Z 14 280-299 98 98 98 166 166 166 T T F <---
112302024-02-15T03:02:21.638Z 15 300-319 99 99 99 167 167 167 F F F
112312024-02-15T03:02:21.638Z 16 320-339 99 99 99 167 167 167 F F F
112322024-02-15T03:02:21.638Z 17 340-359 98 98 98 166 166 166 F F F
112332024-02-15T03:02:21.638Z 18 360-379 89 89 89 153 153 153 F F F
112342024-02-15T03:02:21.638Z 19 380-399 94 94 94 160 160 160 F F F
112352024-02-15T03:02:21.638Z 20 400-419 94 94 94 160 160 160 F F F
112362024-02-15T03:02:21.638Z 21 420-439 96 96 96 163 163 163 F F F
112372024-02-15T03:02:21.639Z 22 440-459 96 96 96 161 161 161 F F F
112382024-02-15T03:02:21.639Z 23 460-479 98 98 98 166 166 166 T T F <---
112392024-02-15T03:02:21.639Z 24 480-499 99 99 99 167 167 167 T T F <---
112402024-02-15T03:02:21.639Z 25 500-519 91 91 91 156 156 156 F F F
112412024-02-15T03:02:21.639Z 26 520-539 98 98 98 166 166 166 T T F <---
112422024-02-15T03:02:21.639Z 27 540-559 96 96 96 162 162 162 T T F <---
112432024-02-15T03:02:21.639Z 28 560-579 96 96 96 162 162 162 F F F
112442024-02-15T03:02:21.639Z 29 580-599 96 96 96 161 161 161 F F F
112452024-02-15T03:02:21.640ZMax gen: 100, Max flush: 168
112462024-02-15T03:02:21.640ZError: Difference in extent metadata found!
112472024-02-15T03:02:21.657ZA Difference in extent metadata is expected here
112482024-02-15T03:02:21.658ZOn loop 50
112492024-02-15T03:02:21.658Z
112502024-02-15T03:02:21.658ZVerifying data now
112512024-02-15T03:02:27.199ZNo change after verify
112522024-02-15T03:02:27.199ZLoop: 50 Downstairs dump after verify (and repair):
112532024-02-15T03:02:27.210ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112542024-02-15T03:02:27.210Z 0 000-019 94 94 94 160 160 160 F F F
112552024-02-15T03:02:27.210Z 1 020-039 101 101 101 169 169 169 F F F
112562024-02-15T03:02:27.210Z 2 040-059 98 98 98 165 165 165 F F F
112572024-02-15T03:02:27.210Z 3 060-079 98 98 98 165 165 165 F F F
112582024-02-15T03:02:27.210Z 4 080-099 101 101 101 169 169 169 F F F
112592024-02-15T03:02:27.210Z 5 100-119 94 94 94 160 160 160 F F F
112602024-02-15T03:02:27.210Z 6 120-139 99 99 99 167 167 167 F F F
112612024-02-15T03:02:27.216Z 7 140-159 100 100 100 168 168 168 F F F
112622024-02-15T03:02:27.216Z 8 160-179 100 100 100 168 168 168 F F F
112632024-02-15T03:02:27.216Z 9 180-199 97 97 97 164 164 164 F F F
112642024-02-15T03:02:27.216Z 10 200-219 98 98 98 165 165 165 F F F
112652024-02-15T03:02:27.216Z 11 220-239 101 101 101 169 169 169 F F F
112662024-02-15T03:02:27.216Z 12 240-259 101 101 101 169 169 169 F F F
112672024-02-15T03:02:27.216Z 13 260-279 97 97 97 164 164 164 F F F
112682024-02-15T03:02:27.217Z 14 280-299 101 101 101 169 169 169 F F F
112692024-02-15T03:02:27.217Z 15 300-319 99 99 99 167 167 167 F F F
112702024-02-15T03:02:27.217Z 16 320-339 99 99 99 167 167 167 F F F
112712024-02-15T03:02:27.217Z 17 340-359 98 98 98 166 166 166 F F F
112722024-02-15T03:02:27.217Z 18 360-379 89 89 89 153 153 153 F F F
112732024-02-15T03:02:27.217Z 19 380-399 94 94 94 160 160 160 F F F
112742024-02-15T03:02:27.217Z 20 400-419 94 94 94 160 160 160 F F F
112752024-02-15T03:02:27.217Z 21 420-439 96 96 96 163 163 163 F F F
112762024-02-15T03:02:27.217Z 22 440-459 96 96 96 161 161 161 F F F
112772024-02-15T03:02:27.217Z 23 460-479 101 101 101 169 169 169 F F F
112782024-02-15T03:02:27.217Z 24 480-499 101 101 101 169 169 169 F F F
112792024-02-15T03:02:27.217Z 25 500-519 91 91 91 156 156 156 F F F
112802024-02-15T03:02:27.217Z 26 520-539 101 101 101 169 169 169 F F F
112812024-02-15T03:02:27.217Z 27 540-559 101 101 101 169 169 169 F F F
112822024-02-15T03:02:27.217Z 28 560-579 96 96 96 162 162 162 F F F
112832024-02-15T03:02:27.217Z 29 580-599 96 96 96 161 161 161 F F F
112842024-02-15T03:02:27.217ZMax gen: 101, Max flush: 169
112852024-02-15T03:02:27.221Z
112862024-02-15T03:02:27.221ZBegin loop 51
112872024-02-15T03:02:27.221ZDownstairs to restart: 2
112882024-02-15T03:02:27.236Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.147611069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546}
112892024-02-15T03:02:27.236ZCrucible runtime is spawned
112902024-02-15T03:02:27.236Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.147985832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546}
112912024-02-15T03:02:27.236Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148022308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546}
112922024-02-15T03:02:27.241Z{"msg":"Crucible 4febac0d-c2f1-46e0-b66e-67414afb673b has session id: d4a6919a-ec39-4fe8-9154-389808c42e8e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148060473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
112932024-02-15T03:02:27.241Z{"msg":"Upstairs opts: Upstairs UUID: 4febac0d-c2f1-46e0-b66e-67414afb673b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148107357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
112942024-02-15T03:02:27.241Z{"msg":"Crucible stats registered with UUID: 4febac0d-c2f1-46e0-b66e-67414afb673b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148147632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
112952024-02-15T03:02:27.241Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148184507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546}
112962024-02-15T03:02:27.241Z{"msg":"4febac0d-c2f1-46e0-b66e-67414afb673b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148207484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
112972024-02-15T03:02:27.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148238471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
112982024-02-15T03:02:27.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148273406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
112992024-02-15T03:02:27.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.14832262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113002024-02-15T03:02:27.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148359505Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113012024-02-15T03:02:27.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.14839918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113022024-02-15T03:02:27.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148417528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113032024-02-15T03:02:27.242Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148439735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113042024-02-15T03:02:27.242Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.14848302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113052024-02-15T03:02:27.242Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148509277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113062024-02-15T03:02:27.242Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:26.148556111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113072024-02-15T03:02:27.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148585067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113082024-02-15T03:02:27.242Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:26.148612934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113092024-02-15T03:02:27.242Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:26.14864349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113102024-02-15T03:02:27.242Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148681645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113112024-02-15T03:02:27.242Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148697543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113122024-02-15T03:02:27.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148711791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113132024-02-15T03:02:27.242Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.148795631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113142024-02-15T03:02:27.242Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.149311546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113152024-02-15T03:02:27.242Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.149340642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113162024-02-15T03:02:27.242Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.14935694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113172024-02-15T03:02:27.242Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.149425272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113182024-02-15T03:02:27.242Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.149882555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113192024-02-15T03:02:27.242Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.150032846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113202024-02-15T03:02:27.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.150374683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113212024-02-15T03:02:27.243Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.150430366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113222024-02-15T03:02:27.243Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.150458222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113232024-02-15T03:02:27.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.150521085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113242024-02-15T03:02:27.243Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.15055751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113252024-02-15T03:02:27.243Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:26.150581987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113262024-02-15T03:02:28.238Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.150405043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113272024-02-15T03:02:28.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.150622855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113282024-02-15T03:02:28.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.151533021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113292024-02-15T03:02:28.257Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.151570717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113302024-02-15T03:02:28.258Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152059325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113312024-02-15T03:02:28.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152639183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113322024-02-15T03:02:28.258Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152673238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113332024-02-15T03:02:28.258Z{"msg":"[0]R flush_numbers[0..12]: [160, 169, 165, 165, 169, 160, 167, 168, 168, 164, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152689196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113342024-02-15T03:02:28.258Z{"msg":"[0]R generation[0..12]: [94, 101, 98, 98, 101, 94, 99, 100, 100, 97, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152706654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113352024-02-15T03:02:28.258Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.15273815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113362024-02-15T03:02:28.258Z{"msg":"[1]R flush_numbers[0..12]: [160, 169, 165, 165, 169, 160, 167, 168, 168, 164, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152754868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113372024-02-15T03:02:28.258Z{"msg":"[1]R generation[0..12]: [94, 101, 98, 98, 101, 94, 99, 100, 100, 97, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152796123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113382024-02-15T03:02:28.259Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.15281611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113392024-02-15T03:02:28.259Z{"msg":"[2]R flush_numbers[0..12]: [160, 169, 165, 165, 169, 160, 167, 168, 168, 164, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152846257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113402024-02-15T03:02:28.259Z{"msg":"[2]R generation[0..12]: [94, 101, 98, 98, 101, 94, 99, 100, 100, 97, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152867034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"Wait for a query_work_queue command to finish before sending IO
113412024-02-15T03:02:28.259Z:1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113422024-02-15T03:02:28.259Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.15290152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113432024-02-15T03:02:28.259Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152933846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113442024-02-15T03:02:28.259Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152952153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113452024-02-15T03:02:28.260Z{"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.152985089Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113462024-02-15T03:02:28.260Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153009056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113472024-02-15T03:02:28.260Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153032453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113482024-02-15T03:02:28.260Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153053331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113492024-02-15T03:02:28.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153067939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113502024-02-15T03:02:28.261Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153099515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113512024-02-15T03:02:28.261Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153129091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113522024-02-15T03:02:28.261Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153147359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113532024-02-15T03:02:28.261Z{"msg":"4febac0d-c2f1-46e0-b66e-67414afb673b is now active with session: d4a6919a-ec39-4fe8-9154-389808c42e8e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153176815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113542024-02-15T03:02:28.261Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.153193353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546,"session_id":"d4a6919a-ec39-4fe8-9154-389808c42e8e"}
113552024-02-15T03:02:28.261Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:27.15322119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546}
113562024-02-15T03:02:28.261ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113572024-02-15T03:02:28.262ZLoading write count information from file "/tmp/test_repair_verify.data"
113582024-02-15T03:02:28.262ZRun Repair workload
113592024-02-15T03:02:28.262Z01/30 Write block 232 len 36864 data: 11 11 13 12 12 10 9 9 9
113602024-02-15T03:02:28.262Z02/30 Read block 257 len 24576
113612024-02-15T03:02:28.262Z03/30 Flush
113622024-02-15T03:02:28.263Z04/30 Write block 290 len 12288 data: 16 14 13
113632024-02-15T03:02:28.263Z05/30 Write block 277 len 24576 data: 9 6 6 6 6 5
113642024-02-15T03:02:28.263Z06/30 Write block 493 len 20480 data: 7 9 10 10 10
113652024-02-15T03:02:28.264Z07/30 Write block 273 len 8192 data: 8 6
113662024-02-15T03:02:28.264Z08/30 Flush
113672024-02-15T03:02:28.264Z09/30 Flush
113682024-02-15T03:02:28.279Z10/30 Write block 525 len 24576 data: 2 4 7 8 8 8
113692024-02-15T03:02:28.279Z11/30 Read block 523 len 28672
113702024-02-15T03:02:28.279Z12/30 Write block 188 len 32768 data: 7 10 9 8 9 12 12 13
113712024-02-15T03:02:28.279Z13/30 Write block 114 len 20480 data: 6 8 7 9 8
113722024-02-15T03:02:28.279Z14/30 Read block 423 len 12288
113732024-02-15T03:02:28.279Z15/30 Read block 504 len 16384
113742024-02-15T03:02:28.279Z16/30 Read block 249 len 4096
113752024-02-15T03:02:28.280Z17/30 Write block 442 len 4096 data: 11
113762024-02-15T03:02:28.280Z18/30 Read block 390 len 16384
113772024-02-15T03:02:28.280Z19/30 Read block 501 len 32768
113782024-02-15T03:02:28.280Z20/30 Write block 199 len 12288 data: 8 8 8
113792024-02-15T03:02:28.280Z21/30 Write block 280 len 4096 data: 7
113802024-02-15T03:02:28.280Z22/30 Flush
113812024-02-15T03:02:28.280Z23/30 Write block 417 len 28672 data: 5 5 6 6 6 6 7
113822024-02-15T03:02:28.280Z24/30 Read block 325 len 12288
113832024-02-15T03:02:28.280Z25/30 Read block 46 len 12288
113842024-02-15T03:02:28.280Z26/30 Read block 349 len 20480
113852024-02-15T03:02:28.280Z27/30 Write block 375 len 36864 data: 7 8 8 9 7 7 9 10 12
113862024-02-15T03:02:28.280Z28/30 Write block 361 len 36864 data: 7 8 6 7 7 9 10 7 6
113872024-02-15T03:02:28.280Z29/30 Read block 9 len 28672
113882024-02-15T03:02:28.281Z30/30 Read block 481 len 32768
113892024-02-15T03:02:28.301Z----------------------------------------------------------------
113902024-02-15T03:02:28.304Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:27
113912024-02-15T03:02:28.304ZStates: Active Active Active
113922024-02-15T03:02:28.305ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
113932024-02-15T03:02:28.305Z 4 Acked 1003 Write 3 Done Done Done false
113942024-02-15T03:02:28.305Z 5 Acked 1004 Write 6 Done Done Done false
113952024-02-15T03:02:28.305Z 6 Acked 1005 Write 5 Done Done Sent false
113962024-02-15T03:02:28.305Z 7 Acked 1006 Write 2 Done Done Sent false
113972024-02-15T03:02:28.306Z 8 Acked 1007 Flush 0 Done Done Sent false
113982024-02-15T03:02:28.306Z 9 Acked 1008 Flush 0 Done Done Sent false
113992024-02-15T03:02:28.306Z 10 Acked 1009 Write 6 Done Done Sent false
114002024-02-15T03:02:28.306Z 11 Acked 1010 Read 7 Done Done Sent false
114012024-02-15T03:02:28.306Z 12 Acked 1011 Write 8 Done Done Sent false
114022024-02-15T03:02:28.306Z 13 Acked 1012 Write 5 Done Done Sent false
114032024-02-15T03:02:28.307Z 14 Acked 1013 Read 3 Done Done Sent false
114042024-02-15T03:02:28.307Z 15 Acked 1014 Read 4 Done Done Sent false
114052024-02-15T03:02:28.307Z 16 Acked 1015 Read 1 Done Done Sent false
114062024-02-15T03:02:28.307Z 17 Acked 1016 Write 1 Done Done Sent false
114072024-02-15T03:02:28.307Z 18 Acked 1017 Read 4 Done Done Sent false
114082024-02-15T03:02:28.307Z 19 Acked 1018 Read 8 Done Done Sent false
114092024-02-15T03:02:28.307Z 20 Acked 1019 Write 3 Done Done Sent false
114102024-02-15T03:02:28.307Z 21 Acked 1020 Write 1 Done Done Sent false
114112024-02-15T03:02:28.308Z 22 Acked 1021 Flush 0 Done Done Sent false
114122024-02-15T03:02:28.308Z 23 Acked 1022 Write 7 Done Done Sent false
114132024-02-15T03:02:28.308Z 24 Acked 1023 Read 3 Done Done Sent false
114142024-02-15T03:02:28.308Z 25 Acked 1024 Read 3 Done Done Sent false
114152024-02-15T03:02:28.308Z 26 Acked 1025 Read 5 Done Done Sent false
114162024-02-15T03:02:28.308Z 27 Acked 1026 Write 9 Done Done Sent false
114172024-02-15T03:02:28.309Z 28 Acked 1027 Write 9 Done Done Sent false
114182024-02-15T03:02:28.309Z 29 Acked 1028 Read 7 Done Done Sent false
114192024-02-15T03:02:28.309Z 30 Acked 1029 Read 8 Done Sent Sent false
114202024-02-15T03:02:28.309Z STATES DS:0 DS:1 DS:2 TOTAL
114212024-02-15T03:02:28.309Z New 0 0 0 0
114222024-02-15T03:02:28.309Z Sent 0 1 25 26
114232024-02-15T03:02:28.309Z Done 27 26 2 55
114242024-02-15T03:02:28.309Z Skipped 0 0 0 0
114252024-02-15T03:02:28.310Z Error 0 0 0 0
114262024-02-15T03:02:28.310ZLast Flush: 1021 1021 1002
114272024-02-15T03:02:28.310ZDownstairs last five completed: 1002 1001 1000
114282024-02-15T03:02:28.310ZUpstairs last five completed: 30 29 28 27 26
114292024-02-15T03:02:28.310Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:27.184064615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1546}
114302024-02-15T03:02:28.310ZWrote out file "/tmp/test_repair_verify.data"
114312024-02-15T03:02:28.310Z
114322024-02-15T03:02:28.310ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114332024-02-15T03:02:28.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114342024-02-15T03:02:28.329Z 0 000-019 94 94 94 160 160 160 F F F
114352024-02-15T03:02:28.329Z 1 020-039 101 101 101 169 169 169 F F F
114362024-02-15T03:02:28.329Z 2 040-059 98 98 98 165 165 165 F F F
114372024-02-15T03:02:28.330Z 3 060-079 98 98 98 165 165 165 F F F
114382024-02-15T03:02:28.330Z 4 080-099 101 101 101 169 169 169 F F F
114392024-02-15T03:02:28.330Z 5 100-119 102 102 94 173 173 160 F F F <---
114402024-02-15T03:02:28.330Z 6 120-139 99 99 99 167 167 167 F F F
114412024-02-15T03:02:28.331Z 7 140-159 100 100 100 168 168 168 F F F
114422024-02-15T03:02:28.331Z 8 160-179 100 100 100 168 168 168 F F F
114432024-02-15T03:02:28.331Z 9 180-199 102 102 97 173 173 164 F F F <---
114442024-02-15T03:02:28.332Z 10 200-219 102 102 98 173 173 165 F F F <---
114452024-02-15T03:02:28.332Z 11 220-239 102 102 102 170 170 170 F F F
114462024-02-15T03:02:28.332Z 12 240-259 102 102 102 170 170 170 F F F
114472024-02-15T03:02:28.332Z 13 260-279 102 102 97 171 171 164 F F T <---
114482024-02-15T03:02:28.332Z 14 280-299 102 102 101 173 173 169 F F T <---
114492024-02-15T03:02:28.332Z 15 300-319 99 99 99 167 167 167 F F F
114502024-02-15T03:02:28.333Z 16 320-339 99 99 99 167 167 167 F F F
114512024-02-15T03:02:28.333Z 17 340-359 98 98 98 166 166 166 F F F
114522024-02-15T03:02:28.333Z 18 360-379 89 89 89 153 153 153 T T F <---
114532024-02-15T03:02:28.334Z 19 380-399 94 94 94 160 160 160 T T F <---
114542024-02-15T03:02:28.334Z 20 400-419 94 94 94 160 160 160 T T F <---
114552024-02-15T03:02:28.334Z 21 420-439 96 96 96 163 163 163 T T F <---
114562024-02-15T03:02:28.334Z 22 440-459 102 102 96 173 173 161 F F F <---
114572024-02-15T03:02:28.334Z 23 460-479 101 101 101 169 169 169 F F F
114582024-02-15T03:02:28.335Z 24 480-499 102 102 101 171 171 169 F F F <---
114592024-02-15T03:02:28.335Z 25 500-519 91 91 91 156 156 156 F F F
114602024-02-15T03:02:28.335Z 26 520-539 102 102 101 173 173 169 F F F <---
114612024-02-15T03:02:28.335Z 27 540-559 101 101 101 169 169 169 F F F
114622024-02-15T03:02:28.335Z 28 560-579 96 96 96 162 162 162 F F F
114632024-02-15T03:02:28.335Z 29 580-599 96 96 96 161 161 161 F F F
114642024-02-15T03:02:28.336ZMax gen: 102, Max flush: 173
114652024-02-15T03:02:28.336ZError: Difference in extent metadata found!
114662024-02-15T03:02:28.336ZA Difference in extent metadata is expected here
114672024-02-15T03:02:28.336ZOn loop 51
114682024-02-15T03:02:28.336Z
114692024-02-15T03:02:28.337ZVerifying data now
114702024-02-15T03:02:33.910ZNo change after verify
114712024-02-15T03:02:33.910ZLoop: 51 Downstairs dump after verify (and repair):
114722024-02-15T03:02:33.926ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114732024-02-15T03:02:33.926Z 0 000-019 94 94 94 160 160 160 F F F
114742024-02-15T03:02:33.926Z 1 020-039 101 101 101 169 169 169 F F F
114752024-02-15T03:02:33.926Z 2 040-059 98 98 98 165 165 165 F F F
114762024-02-15T03:02:33.927Z 3 060-079 98 98 98 165 165 165 F F F
114772024-02-15T03:02:33.927Z 4 080-099 101 101 101 169 169 169 F F F
114782024-02-15T03:02:33.927Z 5 100-119 102 102 102 173 173 173 F F F
114792024-02-15T03:02:33.927Z 6 120-139 99 99 99 167 167 167 F F F
114802024-02-15T03:02:33.927Z 7 140-159 100 100 100 168 168 168 F F F
114812024-02-15T03:02:33.927Z 8 160-179 100 100 100 168 168 168 F F F
114822024-02-15T03:02:33.928Z 9 180-199 102 102 102 173 173 173 F F F
114832024-02-15T03:02:33.928Z 10 200-219 102 102 102 173 173 173 F F F
114842024-02-15T03:02:33.928Z 11 220-239 102 102 102 170 170 170 F F F
114852024-02-15T03:02:33.934Z 12 240-259 102 102 102 170 170 170 F F F
114862024-02-15T03:02:33.934Z 13 260-279 102 102 102 171 171 171 F F F
114872024-02-15T03:02:33.934Z 14 280-299 102 102 102 173 173 173 F F F
114882024-02-15T03:02:33.934Z 15 300-319 99 99 99 167 167 167 F F F
114892024-02-15T03:02:33.934Z 16 320-339 99 99 99 167 167 167 F F F
114902024-02-15T03:02:33.935Z 17 340-359 98 98 98 166 166 166 F F F
114912024-02-15T03:02:33.935Z 18 360-379 103 103 103 174 174 174 F F F
114922024-02-15T03:02:33.935Z 19 380-399 103 103 103 174 174 174 F F F
114932024-02-15T03:02:33.935Z 20 400-419 103 103 103 174 174 174 F F F
114942024-02-15T03:02:33.935Z 21 420-439 103 103 103 174 174 174 F F F
114952024-02-15T03:02:33.935Z 22 440-459 102 102 102 173 173 173 F F F
114962024-02-15T03:02:33.935Z 23 460-479 101 101 101 169 169 169 F F F
114972024-02-15T03:02:33.935Z 24 480-499 102 102 102 171 171 171 F F F
114982024-02-15T03:02:33.935Z 25 500-519 91 91 91 156 156 156 F F F
114992024-02-15T03:02:33.935Z 26 520-539 102 102 102 173 173 173 F F F
115002024-02-15T03:02:33.935Z 27 540-559 101 101 101 169 169 169 F F F
115012024-02-15T03:02:33.935Z 28 560-579 96 96 96 162 162 162 F F F
115022024-02-15T03:02:33.935Z 29 580-599 96 96 96 161 161 161 F F F
115032024-02-15T03:02:33.935ZMax gen: 103, Max flush: 174
115042024-02-15T03:02:33.936Z
115052024-02-15T03:02:33.936ZBegin loop 52
115062024-02-15T03:02:33.936ZDownstairs to restart: 0
115072024-02-15T03:02:33.951Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862073787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554}
115082024-02-15T03:02:33.951Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862334524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554}
115092024-02-15T03:02:33.951Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862352612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554}
115102024-02-15T03:02:33.952Z{"msg":"Crucible 9c07b350-2466-4038-8e33-ad88d00a4f17 has session id: 48babd71-037a-44c8-a371-600981527379","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862375549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115112024-02-15T03:02:33.952Z{"msg":"Upstairs opts: Upstairs UUID: 9c07b350-2466-4038-8e33-ad88d00a4f17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862429012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115122024-02-15T03:02:33.964Z{"msg":"Crucible stats registered with UUID: 9c07b350-2466-4038-8e33-ad88d00a4f17","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862467128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115132024-02-15T03:02:33.964ZCrucible runtime is spawned
115142024-02-15T03:02:33.964Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.86268566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554}
115152024-02-15T03:02:33.964Z{"msg":"9c07b350-2466-4038-8e33-ad88d00a4f17 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862735384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115162024-02-15T03:02:33.964Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862773029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115172024-02-15T03:02:33.964Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862802936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115182024-02-15T03:02:33.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862826523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115192024-02-15T03:02:33.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862866598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115202024-02-15T03:02:33.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862906663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115212024-02-15T03:02:33.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.86292448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115222024-02-15T03:02:33.965Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862950637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115232024-02-15T03:02:33.965Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.862978094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115242024-02-15T03:02:33.965Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.863014029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115252024-02-15T03:02:33.966Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.863119546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115262024-02-15T03:02:33.966Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.863175689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115272024-02-15T03:02:33.966Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:32.863400551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115282024-02-15T03:02:33.966Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:32.863450204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115292024-02-15T03:02:33.966Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:32.863480601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115302024-02-15T03:02:33.966Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.863512457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115312024-02-15T03:02:33.966Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.863537753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115322024-02-15T03:02:33.966Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.863663798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115332024-02-15T03:02:33.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.864021223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115342024-02-15T03:02:33.967Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.864057088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115352024-02-15T03:02:33.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.864218638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115362024-02-15T03:02:33.967Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.864254304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115372024-02-15T03:02:33.967Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.864569774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115382024-02-15T03:02:33.967Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.864678361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115392024-02-15T03:02:33.968Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.865054714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115402024-02-15T03:02:33.968Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.865119915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115412024-02-15T03:02:33.968Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.86515839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115422024-02-15T03:02:33.968Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.8652416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115432024-02-15T03:02:33.968Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.865272176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115442024-02-15T03:02:33.968Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:32.865287854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115452024-02-15T03:02:34.960Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.864916404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115462024-02-15T03:02:34.960Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.864993924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115472024-02-15T03:02:34.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.865835399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115482024-02-15T03:02:34.968Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.865881503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115492024-02-15T03:02:34.968Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.866410767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115502024-02-15T03:02:34.968Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867006382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115512024-02-15T03:02:34.968Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867047337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115522024-02-15T03:02:34.968Z{"msg":"[0]R flush_numbers[0..12]: [160, 169, 165, 165, 169, 173, 167, 168, 168, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867071654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115532024-02-15T03:02:34.968Z{"msg":"[0]R generation[0..12]: [94, 101, 98, 98, 101, 102, 99, 100, 100, 102, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867113838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115542024-02-15T03:02:34.968Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867146794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115552024-02-15T03:02:34.968Z{"msg":"[1]R flush_numbers[0..12]: [160, 169, 165, 165, 169, 173, 167, 168, 168, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867166862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115562024-02-15T03:02:34.968Z{"msg":"[1]R generation[0..12]: [94, 101, 98, 98, 101, 102, 99, 100, 100, 102, 102, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867192669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115572024-02-15T03:02:34.968Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867212236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115582024-02-15T03:02:34.968Z{"msg":"[2]R flush_numbers[0..12]: [160, 169, 165, 165, 169, 173, 167, 168, 168, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867230054Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115592024-02-15T03:02:34.968Z{"msg":"[2]R generation[0..12]: [94, 101, 98, 98, 101, 102, 99, 100, 100, 102, 102, 102]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
115602024-02-15T03:02:34.969Z,"time":"2024-02-15T03:02:33.867256161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115612024-02-15T03:02:34.969Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867277728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115622024-02-15T03:02:34.969Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867302805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115632024-02-15T03:02:34.969Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867320362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115642024-02-15T03:02:34.969Z{"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867352198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115652024-02-15T03:02:34.969Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867369326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115662024-02-15T03:02:34.969Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867386344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115672024-02-15T03:02:34.969Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867402432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115682024-02-15T03:02:34.969Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.86741897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115692024-02-15T03:02:34.969Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867435678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"48babd71-037a-44c8-a371-600981527379"}
115702024-02-15T03:02:34.969Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867460485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115712024-02-15T03:02:34.969Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867476423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115722024-02-15T03:02:34.970Z{"msg":"9c07b350-2466-4038-8e33-ad88d00a4f17 is now active with session: 48babd71-037a-44c8-a371-600981527379","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867492681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115732024-02-15T03:02:34.970Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867515868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554,"session_id":"48babd71-037a-44c8-a371-600981527379"}
115742024-02-15T03:02:34.970Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:33.867531726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554}
115752024-02-15T03:02:34.970ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115762024-02-15T03:02:34.970ZLoading write count information from file "/tmp/test_repair_verify.data"
115772024-02-15T03:02:34.970ZRun Repair workload
115782024-02-15T03:02:34.970Z01/30 Write block 86 len 12288 data: 9 9 7
115792024-02-15T03:02:34.970Z02/30 Read block 463 len 36864
115802024-02-15T03:02:34.970Z03/30 Read block 211 len 16384
115812024-02-15T03:02:34.970Z04/30 Read block 407 len 12288
115822024-02-15T03:02:34.971Z05/30 Flush
115832024-02-15T03:02:34.971Z06/30 Write block 416 len 20480 data: 8 6 6 7 7
115842024-02-15T03:02:34.971Z07/30 Read block 194 len 16384
115852024-02-15T03:02:34.971Z08/30 Read block 330 len 32768
115862024-02-15T03:02:34.971Z09/30 Flush
115872024-02-15T03:02:34.976Z10/30 Write block 407 len 32768 data: 11 12 15 16 16 15 15 12
115882024-02-15T03:02:34.976Z11/30 Read block 46 len 16384
115892024-02-15T03:02:34.976Z12/30 Flush
115902024-02-15T03:02:34.976Z13/30 Write block 219 len 24576 data: 10 11 12 11 9 9
115912024-02-15T03:02:34.976Z14/30 Read block 19 len 24576
115922024-02-15T03:02:34.976Z15/30 Flush
115932024-02-15T03:02:34.976Z16/30 Write block 344 len 40960 data: 8 9 10 9 8 8 9 10 10 10
115942024-02-15T03:02:34.976Z17/30 Write block 492 len 8192 data: 9 8
115952024-02-15T03:02:34.976Z18/30 Write block 106 len 16384 data: 6 5 4 4
115962024-02-15T03:02:34.981Z19/30 Write block 503 len 32768 data: 11 10 9 9 7 8 10 9
115972024-02-15T03:02:34.981Z20/30 Read block 541 len 20480
115982024-02-15T03:02:34.981Z21/30 Read block 109 len 12288
115992024-02-15T03:02:34.981Z22/30 Write block 144 len 4096 data: 10
116002024-02-15T03:02:34.981Z23/30 Read block 416 len 28672
116012024-02-15T03:02:34.988Z24/30 Write block 394 len 24576 data: 8 9 7 6 5 7
116022024-02-15T03:02:34.989Z25/30 Read block 499 len 4096
116032024-02-15T03:02:34.989Z26/30 Write block 388 len 12288 data: 10 11 9
116042024-02-15T03:02:34.989Z27/30 Write block 584 len 16384 data: 9 9 10 8
116052024-02-15T03:02:34.989Z28/30 Write block 576 len 20480 data: 13 13 10 8 10
116062024-02-15T03:02:34.989Z29/30 Read block 269 len 8192
116072024-02-15T03:02:34.989Z30/30 Write block 364 len 40960 data: 8 8 10 11 8 7 7 9 8 9
116082024-02-15T03:02:34.989Z----------------------------------------------------------------
116092024-02-15T03:02:34.990Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
116102024-02-15T03:02:34.990ZStates: Active Active Active
116112024-02-15T03:02:34.990ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116122024-02-15T03:02:34.990Z 1 Acked 1000 Write 3 Done Done Done false
116132024-02-15T03:02:34.990Z 2 Acked 1001 Read 9 Done Done Done false
116142024-02-15T03:02:34.990Z 3 Acked 1002 Read 4 Sent Done Done false
116152024-02-15T03:02:34.991Z 4 Acked 1003 Read 3 Sent Done Done false
116162024-02-15T03:02:34.991Z 5 Acked 1004 Flush 0 Sent Done Done false
116172024-02-15T03:02:34.991Z 6 Acked 1005 Write 5 Sent Done Done false
116182024-02-15T03:02:34.991Z 7 Acked 1006 Read 4 Sent Done Done false
116192024-02-15T03:02:34.991Z 8 Acked 1007 Read 8 Sent Done Done false
116202024-02-15T03:02:34.991Z 9 Acked 1008 Flush 0 Sent Done Done false
116212024-02-15T03:02:34.991Z 10 Acked 1009 Write 8 Sent Done Done false
116222024-02-15T03:02:34.991Z 11 Acked 1010 Read 4 Sent Done Done false
116232024-02-15T03:02:34.992Z 12 Acked 1011 Flush 0 Sent Done Done false
116242024-02-15T03:02:34.992Z 13 Acked 1012 Write 6 Sent Done Done false
116252024-02-15T03:02:34.992Z 14 Acked 1013 Read 6 Sent Done Done false
116262024-02-15T03:02:34.992Z 15 Acked 1014 Flush 0 Sent Done Done false
116272024-02-15T03:02:34.992Z 16 Acked 1015 Write 10 Sent Done Done false
116282024-02-15T03:02:34.992Z 17 Acked 1016 Write 2 Sent Done Done false
116292024-02-15T03:02:34.992Z 18 Acked 1017 Write 4 Sent Done Done false
116302024-02-15T03:02:34.992Z 19 Acked 1018 Write 8 Sent Done Done false
116312024-02-15T03:02:34.993Z 20 Acked 1019 Read 5 Sent Done Done false
116322024-02-15T03:02:34.993Z 21 Acked 1020 Read 3 Sent Done Done false
116332024-02-15T03:02:34.993Z 22 Acked 1021 Write 1 Sent Done Done false
116342024-02-15T03:02:34.993Z 23 Acked 1022 Read 7 Sent Done Done false
116352024-02-15T03:02:34.993Z 24 Acked 1023 Write 6 Sent Done Done false
116362024-02-15T03:02:34.994Z 25 Acked 1024 Read 1 Sent Done Done false
116372024-02-15T03:02:34.994Z 26 Acked 1025 Write 3 Sent Done Done false
116382024-02-15T03:02:34.994Z 27 Acked 1026 Write 4 Sent Done Done false
116392024-02-15T03:02:34.994Z 28 Acked 1027 Write 5 Sent Done Done false
116402024-02-15T03:02:34.994Z 29 Acked 1028 Read 2 Sent Done Done false
116412024-02-15T03:02:34.995Z 30 Acked 1029 Write 10 Sent Sent Sent false
116422024-02-15T03:02:34.995Z STATES DS:0 DS:1 DS:2 TOTAL
116432024-02-15T03:02:34.995Z New 0 0 0 0
116442024-02-15T03:02:34.995Z Sent 28 1 1 30
116452024-02-15T03:02:34.995Z Done 2 29 29 60
116462024-02-15T03:02:34.995Z Skipped 0 0 0 0
116472024-02-15T03:02:34.996Z Error 0 0 0 0
116482024-02-15T03:02:34.996ZLast Flush: 0 1014 1014
116492024-02-15T03:02:34.996ZDownstairs last five completed:
116502024-02-15T03:02:34.996ZUpstairs last five completed: 30 29 28 27 26
116512024-02-15T03:02:35.003Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:33.899410902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1554}
116522024-02-15T03:02:35.003ZWrote out file "/tmp/test_repair_verify.data"
116532024-02-15T03:02:35.003Z
116542024-02-15T03:02:35.003ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
116552024-02-15T03:02:35.031ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116562024-02-15T03:02:35.031Z 0 000-019 94 94 94 160 160 160 F F F
116572024-02-15T03:02:35.031Z 1 020-039 101 101 101 169 169 169 F F F
116582024-02-15T03:02:35.031Z 2 040-059 98 98 98 165 165 165 F F F
116592024-02-15T03:02:35.031Z 3 060-079 98 98 98 165 165 165 F F F
116602024-02-15T03:02:35.031Z 4 080-099 101 104 104 169 175 175 T F F <---
116612024-02-15T03:02:35.031Z 5 100-119 102 102 102 173 173 173 F T T <---
116622024-02-15T03:02:35.031Z 6 120-139 99 99 99 167 167 167 F F F
116632024-02-15T03:02:35.032Z 7 140-159 100 100 100 168 168 168 F T T <---
116642024-02-15T03:02:35.038Z 8 160-179 100 100 100 168 168 168 F F F
116652024-02-15T03:02:35.039Z 9 180-199 102 102 102 173 173 173 F F F
116662024-02-15T03:02:35.039Z 10 200-219 102 104 104 173 178 178 F F F <---
116672024-02-15T03:02:35.039Z 11 220-239 102 104 104 170 178 178 F F F <---
116682024-02-15T03:02:35.039Z 12 240-259 102 102 102 170 170 170 F F F
116692024-02-15T03:02:35.040Z 13 260-279 102 102 102 171 171 171 F F F
116702024-02-15T03:02:35.040Z 14 280-299 102 102 102 173 173 173 F F F
116712024-02-15T03:02:35.040Z 15 300-319 99 99 99 167 167 167 F F F
116722024-02-15T03:02:35.040Z 16 320-339 99 99 99 167 167 167 F F F
116732024-02-15T03:02:35.040Z 17 340-359 98 98 98 166 166 166 F T T <---
116742024-02-15T03:02:35.040Z 18 360-379 103 103 103 174 174 174 F T T <---
116752024-02-15T03:02:35.041Z 19 380-399 103 103 103 174 174 174 F T T <---
116762024-02-15T03:02:35.041Z 20 400-419 103 104 104 174 177 177 F F F <---
116772024-02-15T03:02:35.041Z 21 420-439 103 104 104 174 176 176 F F F <---
116782024-02-15T03:02:35.041Z 22 440-459 102 102 102 173 173 173 F F F
116792024-02-15T03:02:35.041Z 23 460-479 101 101 101 169 169 169 F F F
116802024-02-15T03:02:35.041Z 24 480-499 102 102 102 171 171 171 F T T <---
116812024-02-15T03:02:35.041Z 25 500-519 91 91 91 156 156 156 F T T <---
116822024-02-15T03:02:35.042Z 26 520-539 102 102 102 173 173 173 F F F
116832024-02-15T03:02:35.042Z 27 540-559 101 101 101 169 169 169 F F F
116842024-02-15T03:02:35.042Z 28 560-579 96 96 96 162 162 162 F T T <---
116852024-02-15T03:02:35.042Z 29 580-599 96 96 96 161 161 161 F T T <---
116862024-02-15T03:02:35.042ZMax gen: 104, Max flush: 178
116872024-02-15T03:02:35.042ZError: Difference in extent metadata found!
116882024-02-15T03:02:35.043ZA Difference in extent metadata is expected here
116892024-02-15T03:02:35.043ZOn loop 52
116902024-02-15T03:02:35.043Z
116912024-02-15T03:02:35.043ZVerifying data now
116922024-02-15T03:02:40.721ZNo change after verify
116932024-02-15T03:02:40.721ZLoop: 52 Downstairs dump after verify (and repair):
116942024-02-15T03:02:40.742ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116952024-02-15T03:02:40.742Z 0 000-019 94 94 94 160 160 160 F F F
116962024-02-15T03:02:40.742Z 1 020-039 101 101 101 169 169 169 F F F
116972024-02-15T03:02:40.742Z 2 040-059 98 98 98 165 165 165 F F F
116982024-02-15T03:02:40.742Z 3 060-079 98 98 98 165 165 165 F F F
116992024-02-15T03:02:40.742Z 4 080-099 104 104 104 175 175 175 F F F
117002024-02-15T03:02:40.743Z 5 100-119 105 105 105 179 179 179 F F F
117012024-02-15T03:02:40.743Z 6 120-139 99 99 99 167 167 167 F F F
117022024-02-15T03:02:40.743Z 7 140-159 105 105 105 179 179 179 F F F
117032024-02-15T03:02:40.743Z 8 160-179 100 100 100 168 168 168 F F F
117042024-02-15T03:02:40.743Z 9 180-199 102 102 102 173 173 173 F F F
117052024-02-15T03:02:40.743Z 10 200-219 104 104 104 178 178 178 F F F
117062024-02-15T03:02:40.743Z 11 220-239 104 104 104 178 178 178 F F F
117072024-02-15T03:02:40.744Z 12 240-259 102 102 102 170 170 170 F F F
117082024-02-15T03:02:40.744Z 13 260-279 102 102 102 171 171 171 F F F
117092024-02-15T03:02:40.744Z 14 280-299 102 102 102 173 173 173 F F F
117102024-02-15T03:02:40.744Z 15 300-319 99 99 99 167 167 167 F F F
117112024-02-15T03:02:40.744Z 16 320-339 99 99 99 167 167 167 F F F
117122024-02-15T03:02:40.744Z 17 340-359 105 105 105 179 179 179 F F F
117132024-02-15T03:02:40.755Z 18 360-379 105 105 105 179 179 179 F F F
117142024-02-15T03:02:40.755Z 19 380-399 105 105 105 179 179 179 F F F
117152024-02-15T03:02:40.755Z 20 400-419 104 104 104 177 177 177 F F F
117162024-02-15T03:02:40.755Z 21 420-439 104 104 104 176 176 176 F F F
117172024-02-15T03:02:40.756Z 22 440-459 102 102 102 173 173 173 F F F
117182024-02-15T03:02:40.756Z 23 460-479 101 101 101 169 169 169 F F F
117192024-02-15T03:02:40.756Z 24 480-499 105 105 105 179 179 179 F F F
117202024-02-15T03:02:40.756Z 25 500-519 105 105 105 179 179 179 F F F
117212024-02-15T03:02:40.756Z 26 520-539 102 102 102 173 173 173 F F F
117222024-02-15T03:02:40.756Z 27 540-559 101 101 101 169 169 169 F F F
117232024-02-15T03:02:40.756Z 28 560-579 105 105 105 179 179 179 F F F
117242024-02-15T03:02:40.756Z 29 580-599 105 105 105 179 179 179 F F F
117252024-02-15T03:02:40.756ZMax gen: 105, Max flush: 179
117262024-02-15T03:02:40.756Z
117272024-02-15T03:02:40.756ZBegin loop 53
117282024-02-15T03:02:40.756ZDownstairs to restart: 0
117292024-02-15T03:02:40.769Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.675536622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562}
117302024-02-15T03:02:40.769Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.675851712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562}
117312024-02-15T03:02:40.769ZCrucible runtime is spawned
117322024-02-15T03:02:40.770Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.675892817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562}
117332024-02-15T03:02:40.770Z{"msg":"Crucible 16f2a27d-7e46-4030-af39-7078b5142988 has session id: 3c16dd54-934a-409f-aec5-474fc8f9acab","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.675953919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117342024-02-15T03:02:40.770Z{"msg":"Upstairs opts: Upstairs UUID: 16f2a27d-7e46-4030-af39-7078b5142988, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676001853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117352024-02-15T03:02:40.770Z{"msg":"Crucible stats registered with UUID: 16f2a27d-7e46-4030-af39-7078b5142988","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676022431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117362024-02-15T03:02:40.770Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676049107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562}
117372024-02-15T03:02:40.770Z{"msg":"16f2a27d-7e46-4030-af39-7078b5142988 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676099591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117382024-02-15T03:02:40.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676129377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117392024-02-15T03:02:40.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676160193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117402024-02-15T03:02:40.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676178831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117412024-02-15T03:02:40.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676219796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117422024-02-15T03:02:40.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676278038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117432024-02-15T03:02:40.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676329072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117442024-02-15T03:02:40.771Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676367927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117452024-02-15T03:02:40.771Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676419841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117462024-02-15T03:02:40.771Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676453776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117472024-02-15T03:02:40.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676485083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117482024-02-15T03:02:40.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676514999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117492024-02-15T03:02:40.771Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:39.676544465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117502024-02-15T03:02:40.771Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:39.676572852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117512024-02-15T03:02:40.771Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:39.676621035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117522024-02-15T03:02:40.771Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676654211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117532024-02-15T03:02:40.771Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676671259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117542024-02-15T03:02:40.772Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.676773996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117552024-02-15T03:02:40.772Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.677201423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117562024-02-15T03:02:40.772Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.677231229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117572024-02-15T03:02:40.772Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.677247597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117582024-02-15T03:02:40.772Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.677278443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117592024-02-15T03:02:40.778Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.677759223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117602024-02-15T03:02:40.778Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.677923572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117612024-02-15T03:02:40.778Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.678304175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117622024-02-15T03:02:40.778Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.67834091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117632024-02-15T03:02:40.779Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.678363447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117642024-02-15T03:02:40.779Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.678392684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117652024-02-15T03:02:40.779Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.678427309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117662024-02-15T03:02:40.779Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:39.678447837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117672024-02-15T03:02:42.181Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.678189662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117682024-02-15T03:02:42.188Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.678319326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117692024-02-15T03:02:42.188Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.679279216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117702024-02-15T03:02:42.188Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.679317051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117712024-02-15T03:02:42.188Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.679890169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117722024-02-15T03:02:42.188Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680476856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117732024-02-15T03:02:42.188Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680515271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117742024-02-15T03:02:42.188Z{"msg":"[0]R flush_numbers[0..12]: [160, 169, 165, 165, 175, 179, 167, 179, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680534279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117752024-02-15T03:02:42.188Z{"msg":"[0]R generation[0..12]: [94, 101, 98, 98, 104, 105, 99, 105, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680565375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117762024-02-15T03:02:42.188Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680584642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117772024-02-15T03:02:42.188Z{"msg":"[1]R flush_numbers[0..12]: [160, 169, 165, 165, 175, 179, 167, 179, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680611419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117782024-02-15T03:02:42.188Z{"msg":"[1]R generation[0..12]: [94, 101, 98, 98, 104, 105, 99, 105, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680628907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117792024-02-15T03:02:42.189Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680646725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117802024-02-15T03:02:42.189Z{"msg":"[2]R flush_numbers[0..12]: [160, 169, 165, 165, 175, 179, 167, 179, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680673041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117812024-02-15T03:02:42.189Z{"msg":"[2]R generation[0..12]: [94, 101, 98, 98, 104, 105, 99, 105, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680690669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117822024-02-15T03:02:42.189Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
117832024-02-15T03:02:42.189Z,"time":"2024-02-15T03:02:40.680716686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117842024-02-15T03:02:42.189Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680735634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117852024-02-15T03:02:42.189Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680753861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117862024-02-15T03:02:42.189Z{"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680770909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117872024-02-15T03:02:42.189Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680811654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117882024-02-15T03:02:42.189Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680854699Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117892024-02-15T03:02:42.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680883675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117902024-02-15T03:02:42.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.68092337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117912024-02-15T03:02:42.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680954846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117922024-02-15T03:02:42.189Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.680993921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117932024-02-15T03:02:42.189Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.681024477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117942024-02-15T03:02:42.189Z{"msg":"16f2a27d-7e46-4030-af39-7078b5142988 is now active with session: 3c16dd54-934a-409f-aec5-474fc8f9acab","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.681051074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117952024-02-15T03:02:42.189Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:40.681091369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562,"session_id":"3c16dd54-934a-409f-aec5-474fc8f9acab"}
117962024-02-15T03:02:42.189Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
117972024-02-15T03:02:42.189Z,"time":"2024-02-15T03:02:40.681115136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562}
117982024-02-15T03:02:42.189ZLoading write count information from file "/tmp/test_repair_verify.data"
117992024-02-15T03:02:42.189ZRun Repair workload
118002024-02-15T03:02:42.189Z01/30 Write block 279 len 36864 data: 7 8 7 6 9 11 13 13 12
118012024-02-15T03:02:42.189Z02/30 Write block 156 len 4096 data: 13
118022024-02-15T03:02:42.190Z03/30 Flush
118032024-02-15T03:02:42.190Z04/30 Write block 490 len 12288 data: 9 9 10
118042024-02-15T03:02:42.190Z05/30 Write block 529 len 20480 data: 9 9 5 8 6
118052024-02-15T03:02:42.190Z06/30 Read block 252 len 36864
118062024-02-15T03:02:42.190Z07/30 Write block 325 len 24576 data: 8 9 8 9 9 9
118072024-02-15T03:02:42.190Z08/30 Read block 546 len 8192
118082024-02-15T03:02:42.190Z09/30 Read block 426 len 36864
118092024-02-15T03:02:42.190Z10/30 Read block 240 len 12288
118102024-02-15T03:02:42.190Z11/30 Read block 508 len 40960
118112024-02-15T03:02:42.190Z12/30 Write block 529 len 20480 data: 10 10 6 9 7
118122024-02-15T03:02:42.190Z13/30 Read block 560 len 4096
118132024-02-15T03:02:42.190Z14/30 Write block 284 len 4096 data: 12
118142024-02-15T03:02:42.190Z15/30 Read block 56 len 8192
118152024-02-15T03:02:42.190Z16/30 Read block 568 len 20480
118162024-02-15T03:02:42.190Z17/30 Write block 554 len 20480 data: 9 9 7 11 12
118172024-02-15T03:02:42.190Z18/30 Write block 588 len 36864 data: 7 10 12 14 11 12 12 10 7
118182024-02-15T03:02:42.190Z19/30 Write block 380 len 32768 data: 8 10 11 13 14 12 11 11
118192024-02-15T03:02:42.191Z20/30 Write block 445 len 20480 data: 10 10 9 8 7
118202024-02-15T03:02:42.191Z21/30 Read block 336 len 24576
118212024-02-15T03:02:42.191Z22/30 Read block 271 len 16384
118222024-02-15T03:02:42.191Z23/30 Write block 571 len 40960 data: 8 8 12 13 12 14 14 11 9 11
118232024-02-15T03:02:42.192Z24/30 Flush
118242024-02-15T03:02:42.192Z25/30 Write block 559 len 28672 data: 13 12 8 8 8 9 9
118252024-02-15T03:02:42.192Z26/30 Read block 228 len 20480
118262024-02-15T03:02:42.192Z27/30 Read block 37 len 36864
118272024-02-15T03:02:42.192Z28/30 Read block 245 len 20480
118282024-02-15T03:02:42.192Z29/30 Read block 404 len 4096
118292024-02-15T03:02:42.192Z30/30 Read block 288 len 4096
118302024-02-15T03:02:42.192Z----------------------------------------------------------------
118312024-02-15T03:02:42.193Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:27
118322024-02-15T03:02:42.193ZStates: Active Active Active
118332024-02-15T03:02:42.193ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118342024-02-15T03:02:42.193Z 4 Acked 1003 Write 3 Sent Done Done false
118352024-02-15T03:02:42.193Z 5 Acked 1004 Write 5 Sent Done Done false
118362024-02-15T03:02:42.193Z 6 Acked 1005 Read 9 Sent Done Done false
118372024-02-15T03:02:42.193Z 7 Acked 1006 Write 6 Sent Done Done false
118382024-02-15T03:02:42.193Z 8 Acked 1007 Read 2 Sent Done Done false
118392024-02-15T03:02:42.194Z 9 Acked 1008 Read 9 Sent Done Done false
118402024-02-15T03:02:42.194Z 10 Acked 1009 Read 3 Sent Done Done false
118412024-02-15T03:02:42.194Z 11 Acked 1010 Read 10 Sent Done Done false
118422024-02-15T03:02:42.194Z 12 Acked 1011 Write 5 Sent Done Done false
118432024-02-15T03:02:42.194Z 13 Acked 1012 Read 1 Sent Done Done false
118442024-02-15T03:02:42.194Z 14 Acked 1013 Write 1 Sent Done Done false
118452024-02-15T03:02:42.194Z 15 Acked 1014 Read 2 Sent Done Done false
118462024-02-15T03:02:42.194Z 16 Acked 1015 Read 5 Sent Done Done false
118472024-02-15T03:02:42.195Z 17 Acked 1016 Write 5 Sent Done Done false
118482024-02-15T03:02:42.195Z 18 Acked 1017 Write 9 Sent Done Done false
118492024-02-15T03:02:42.195Z 19 Acked 1018 Write 8 Sent Done Done false
118502024-02-15T03:02:42.195Z 20 Acked 1019 Write 5 Sent Done Done false
118512024-02-15T03:02:42.195Z 21 Acked 1020 Read 6 Sent Done Done false
118522024-02-15T03:02:42.195Z 22 Acked 1021 Read 4 Sent Done Done false
118532024-02-15T03:02:42.195Z 23 Acked 1022 Write 10 Sent Done Done false
118542024-02-15T03:02:42.196Z 24 Acked 1023 Flush 0 Sent Done Done false
118552024-02-15T03:02:42.196Z 25 Acked 1024 Write 7 Sent Done Done false
118562024-02-15T03:02:42.196Z 26 Acked 1025 Read 5 Sent Done Done false
118572024-02-15T03:02:42.196Z 27 Acked 1026 Read 9 Sent Done Done false
118582024-02-15T03:02:42.196Z 28 Acked 1027 Read 5 Sent Done Done false
118592024-02-15T03:02:42.196Z 29 Acked 1028 Read 1 Sent Done Done false
118602024-02-15T03:02:42.197Z 30 Acked 1029 Read 1 Sent Done Done false
118612024-02-15T03:02:42.197Z STATES DS:0 DS:1 DS:2 TOTAL
118622024-02-15T03:02:42.197Z New 0 0 0 0
118632024-02-15T03:02:42.197Z Sent 27 0 0 27
118642024-02-15T03:02:42.197Z Done 0 27 27 54
118652024-02-15T03:02:42.198Z Skipped 0 0 0 0
118662024-02-15T03:02:42.198Z Error 0 0 0 0
118672024-02-15T03:02:42.198ZLast Flush: 1002 1023 1023
118682024-02-15T03:02:42.233ZDownstairs last five completed: 1002 1001 1000
118692024-02-15T03:02:42.233ZUpstairs last five completed: 30 29 28 27 26
118702024-02-15T03:02:42.234Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:40.712839271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1562}
118712024-02-15T03:02:42.234ZWrote out file "/tmp/test_repair_verify.data"
118722024-02-15T03:02:42.234Z
118732024-02-15T03:02:42.234ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
118742024-02-15T03:02:42.234ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118752024-02-15T03:02:42.234Z 0 000-019 94 94 94 160 160 160 F F F
118762024-02-15T03:02:42.234Z 1 020-039 101 101 101 169 169 169 F F F
118772024-02-15T03:02:42.234Z 2 040-059 98 98 98 165 165 165 F F F
118782024-02-15T03:02:42.234Z 3 060-079 98 98 98 165 165 165 F F F
118792024-02-15T03:02:42.234Z 4 080-099 104 104 104 175 175 175 F F F
118802024-02-15T03:02:42.234Z 5 100-119 105 105 105 179 179 179 F F F
118812024-02-15T03:02:42.234Z 6 120-139 99 99 99 167 167 167 F F F
118822024-02-15T03:02:42.234Z 7 140-159 106 106 106 180 180 180 F F F
118832024-02-15T03:02:42.234Z 8 160-179 100 100 100 168 168 168 F F F
118842024-02-15T03:02:42.234Z 9 180-199 102 102 102 173 173 173 F F F
118852024-02-15T03:02:42.235Z 10 200-219 104 104 104 178 178 178 F F F
118862024-02-15T03:02:42.235Z 11 220-239 104 104 104 178 178 178 F F F
118872024-02-15T03:02:42.235Z 12 240-259 102 102 102 170 170 170 F F F
118882024-02-15T03:02:42.235Z 13 260-279 106 106 106 180 180 180 F F F
118892024-02-15T03:02:42.235Z 14 280-299 106 106 106 180 181 181 F F F <---
118902024-02-15T03:02:42.235Z 15 300-319 99 99 99 167 167 167 F F F
118912024-02-15T03:02:42.235Z 16 320-339 99 106 106 167 181 181 F F F <---
118922024-02-15T03:02:42.235Z 17 340-359 105 105 105 179 179 179 F F F
118932024-02-15T03:02:42.235Z 18 360-379 105 105 105 179 179 179 F F F
118942024-02-15T03:02:42.235Z 19 380-399 105 106 106 179 181 181 F F F <---
118952024-02-15T03:02:42.235Z 20 400-419 104 104 104 177 177 177 F F F
118962024-02-15T03:02:42.235Z 21 420-439 104 104 104 176 176 176 F F F
118972024-02-15T03:02:42.235Z 22 440-459 102 106 106 173 181 181 F F F <---
118982024-02-15T03:02:42.235Z 23 460-479 101 101 101 169 169 169 F F F
118992024-02-15T03:02:42.236Z 24 480-499 105 106 106 179 181 181 F F F <---
119002024-02-15T03:02:42.236Z 25 500-519 105 105 105 179 179 179 F F F
119012024-02-15T03:02:42.236Z 26 520-539 102 106 106 173 181 181 F F F <---
119022024-02-15T03:02:42.236Z 27 540-559 101 106 106 169 181 181 F T T <---
119032024-02-15T03:02:42.236Z 28 560-579 105 106 106 179 181 181 F T T <---
119042024-02-15T03:02:42.236Z 29 580-599 105 106 106 179 181 181 F F F <---
119052024-02-15T03:02:42.236ZMax gen: 106, Max flush: 181
119062024-02-15T03:02:42.236ZError: Difference in extent metadata found!
119072024-02-15T03:02:42.236ZA Difference in extent metadata is expected here
119082024-02-15T03:02:42.236ZOn loop 53
119092024-02-15T03:02:42.236Z
119102024-02-15T03:02:42.236ZVerifying data now
119112024-02-15T03:02:46.313ZNo change after verify
119122024-02-15T03:02:46.313ZLoop: 53 Downstairs dump after verify (and repair):
119132024-02-15T03:02:46.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119142024-02-15T03:02:46.328Z 0 000-019 94 94 94 160 160 160 F F F
119152024-02-15T03:02:46.328Z 1 020-039 101 101 101 169 169 169 F F F
119162024-02-15T03:02:46.328Z 2 040-059 98 98 98 165 165 165 F F F
119172024-02-15T03:02:46.328Z 3 060-079 98 98 98 165 165 165 F F F
119182024-02-15T03:02:46.328Z 4 080-099 104 104 104 175 175 175 F F F
119192024-02-15T03:02:46.329Z 5 100-119 105 105 105 179 179 179 F F F
119202024-02-15T03:02:46.329Z 6 120-139 99 99 99 167 167 167 F F F
119212024-02-15T03:02:46.334Z 7 140-159 106 106 106 180 180 180 F F F
119222024-02-15T03:02:46.335Z 8 160-179 100 100 100 168 168 168 F F F
119232024-02-15T03:02:46.335Z 9 180-199 102 102 102 173 173 173 F F F
119242024-02-15T03:02:46.335Z 10 200-219 104 104 104 178 178 178 F F F
119252024-02-15T03:02:46.335Z 11 220-239 104 104 104 178 178 178 F F F
119262024-02-15T03:02:46.335Z 12 240-259 102 102 102 170 170 170 F F F
119272024-02-15T03:02:46.335Z 13 260-279 106 106 106 180 180 180 F F F
119282024-02-15T03:02:46.335Z 14 280-299 106 106 106 181 181 181 F F F
119292024-02-15T03:02:46.335Z 15 300-319 99 99 99 167 167 167 F F F
119302024-02-15T03:02:46.336Z 16 320-339 106 106 106 181 181 181 F F F
119312024-02-15T03:02:46.336Z 17 340-359 105 105 105 179 179 179 F F F
119322024-02-15T03:02:46.336Z 18 360-379 105 105 105 179 179 179 F F F
119332024-02-15T03:02:46.336Z 19 380-399 106 106 106 181 181 181 F F F
119342024-02-15T03:02:46.336Z 20 400-419 104 104 104 177 177 177 F F F
119352024-02-15T03:02:46.336Z 21 420-439 104 104 104 176 176 176 F F F
119362024-02-15T03:02:46.336Z 22 440-459 106 106 106 181 181 181 F F F
119372024-02-15T03:02:46.336Z 23 460-479 101 101 101 169 169 169 F F F
119382024-02-15T03:02:46.336Z 24 480-499 106 106 106 181 181 181 F F F
119392024-02-15T03:02:46.336Z 25 500-519 105 105 105 179 179 179 F F F
119402024-02-15T03:02:46.336Z 26 520-539 106 106 106 181 181 181 F F F
119412024-02-15T03:02:46.337Z 27 540-559 107 107 107 182 182 182 F F F
119422024-02-15T03:02:46.337Z 28 560-579 107 107 107 182 182 182 F F F
119432024-02-15T03:02:46.337Z 29 580-599 106 106 106 181 181 181 F F F
119442024-02-15T03:02:46.337ZMax gen: 107, Max flush: 182
119452024-02-15T03:02:46.337Z
119462024-02-15T03:02:46.337ZBegin loop 54
119472024-02-15T03:02:46.338ZDownstairs to restart: 1
119482024-02-15T03:02:46.349Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.260659123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570}
119492024-02-15T03:02:46.349Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.260931838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570}
119502024-02-15T03:02:46.349Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.260959055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570}
119512024-02-15T03:02:46.357Z{"msg":"Crucible 38a38394-df4f-4197-a97a-a0bb36297e33 has session id: 0f758a3b-87d8-495b-8027-db89a11ca1a0","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.260990581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119522024-02-15T03:02:46.357ZCrucible runtime is spawned
119532024-02-15T03:02:46.357Z{"msg":"Upstairs opts: Upstairs UUID: 38a38394-df4f-4197-a97a-a0bb36297e33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261046564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119542024-02-15T03:02:46.357Z{"msg":"Crucible stats registered with UUID: 38a38394-df4f-4197-a97a-a0bb36297e33","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261093208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119552024-02-15T03:02:46.357Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261114966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570}
119562024-02-15T03:02:46.358Z{"msg":"38a38394-df4f-4197-a97a-a0bb36297e33 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261249889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119572024-02-15T03:02:46.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261278655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119582024-02-15T03:02:46.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261305042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119592024-02-15T03:02:46.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261341677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119602024-02-15T03:02:46.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261381202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119612024-02-15T03:02:46.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261419148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119622024-02-15T03:02:46.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261446384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119632024-02-15T03:02:46.358Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261485609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119642024-02-15T03:02:46.358Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261520175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119652024-02-15T03:02:46.358Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261544642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119662024-02-15T03:02:46.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261664787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119672024-02-15T03:02:46.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261735968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119682024-02-15T03:02:46.358Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.261774573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119692024-02-15T03:02:46.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.262413553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119702024-02-15T03:02:46.359Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.262471606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119712024-02-15T03:02:46.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.26251264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119722024-02-15T03:02:46.359Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.262547016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119732024-02-15T03:02:46.359Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263056822Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119742024-02-15T03:02:46.359Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263106166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119752024-02-15T03:02:46.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263169878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119762024-02-15T03:02:46.359Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263214872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119772024-02-15T03:02:46.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263692063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119782024-02-15T03:02:46.359Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263728978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119792024-02-15T03:02:46.359Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263745466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119802024-02-15T03:02:46.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263766403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119812024-02-15T03:02:46.359Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263800899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119822024-02-15T03:02:46.359Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263831855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119832024-02-15T03:02:46.359Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.263939192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119842024-02-15T03:02:46.360Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264403614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119852024-02-15T03:02:46.360Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264440659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119862024-02-15T03:02:46.360Z{"msg":"[0]R flush_numbers[0..12]: [160, 169, 165, 165, 175, 179, 167, 180, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264465466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119872024-02-15T03:02:46.360Z{"msg":"[0]R generation[0..12]: [94, 101, 98, 98, 104, 105, 99, 106, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264500391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119882024-02-15T03:02:46.360Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264517869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119892024-02-15T03:02:46.360Z{"msg":"[1]R flush_numbers[0..12]: [160, 169, 165, 165, 175, 179, 167, 180, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264532447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119902024-02-15T03:02:46.360Z{"msg":"[1]R generation[0..12]: [94, 101, 98, 98, 104, 105, 99, 106, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264547336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119912024-02-15T03:02:46.360Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264569963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119922024-02-15T03:02:46.360Z{"msg":"[2]R flush_numbers[0..12]: [160, 169, 165, 165, 175, 179, 167, 180, 168, 173, 178, 178]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
119932024-02-15T03:02:46.360Z,"time":"2024-02-15T03:02:45.264584701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119942024-02-15T03:02:46.360Z{"msg":"[2]R generation[0..12]: [94, 101, 98, 98, 104, 105, 99, 106, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264603198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119952024-02-15T03:02:46.360Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264617757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119962024-02-15T03:02:46.360Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264637894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119972024-02-15T03:02:46.360Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264652003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119982024-02-15T03:02:46.360Z{"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264676999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
119992024-02-15T03:02:46.361Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264690747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120002024-02-15T03:02:46.361Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264704476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120012024-02-15T03:02:46.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264745481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120022024-02-15T03:02:46.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264766558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120032024-02-15T03:02:46.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264788395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120042024-02-15T03:02:46.361Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264802334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120052024-02-15T03:02:46.361Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264816332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120062024-02-15T03:02:46.361Z{"msg":"38a38394-df4f-4197-a97a-a0bb36297e33 is now active with session: 0f758a3b-87d8-495b-8027-db89a11ca1a0","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.2648295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120072024-02-15T03:02:46.361Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264848728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570,"session_id":"0f758a3b-87d8-495b-8027-db89a11ca1a0"}
120082024-02-15T03:02:46.361Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:45.264862906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570}
120092024-02-15T03:02:46.361ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120102024-02-15T03:02:46.361ZLoading write count information from file "/tmp/test_repair_verify.data"
120112024-02-15T03:02:46.361ZRun Repair workload
120122024-02-15T03:02:46.365Z01/30 Write block 271 len 32768 data: 7 9 9 7 7 7 10 7
120132024-02-15T03:02:46.365Z02/30 Read block 263 len 32768
120142024-02-15T03:02:46.365Z03/30 Read block 414 len 40960
120152024-02-15T03:02:46.365Z04/30 Read block 297 len 4096
120162024-02-15T03:02:46.365Z05/30 Read block 561 len 24576
120172024-02-15T03:02:46.365Z06/30 Read block 114 len 4096
120182024-02-15T03:02:46.366Z07/30 Read block 589 len 24576
120192024-02-15T03:02:46.366Z08/30 Read block 319 len 32768
120202024-02-15T03:02:46.366Z09/30 Read block 84 len 40960
120212024-02-15T03:02:46.366Z10/30 Flush
120222024-02-15T03:02:46.370Z11/30 Write block 402 len 40960 data: 4 4 4 9 10 12 13 16 17 17
120232024-02-15T03:02:46.370Z12/30 Write block 279 len 8192 data: 8 9
120242024-02-15T03:02:46.370Z13/30 Write block 79 len 8192 data: 6 5
120252024-02-15T03:02:46.370Z14/30 Write block 489 len 8192 data: 9 10
120262024-02-15T03:02:46.370Z15/30 Read block 101 len 20480
120272024-02-15T03:02:46.371Z16/30 Read block 530 len 28672
120282024-02-15T03:02:46.371Z17/30 Write block 330 len 36864 data: 10 11 12 12 14 14 12 12 12
120292024-02-15T03:02:46.371Z18/30 Read block 580 len 36864
120302024-02-15T03:02:46.374Z19/30 Read block 412 len 28672
120312024-02-15T03:02:46.374Z20/30 Flush
120322024-02-15T03:02:46.378Z21/30 Write block 455 len 32768 data: 6 6 6 6 6 9 9 8
120332024-02-15T03:02:46.378Z22/30 Read block 588 len 32768
120342024-02-15T03:02:46.378Z23/30 Write block 574 len 24576 data: 14 13 15 15 12 10
120352024-02-15T03:02:46.378Z24/30 Read block 174 len 24576
120362024-02-15T03:02:46.381Z25/30 Read block 31 len 28672
120372024-02-15T03:02:46.381Z26/30 Write block 249 len 32768 data: 8 8 8 9 8 8 7 6
120382024-02-15T03:02:46.381Z27/30 Read block 321 len 40960
120392024-02-15T03:02:46.385Z28/30 Read block 162 len 36864
120402024-02-15T03:02:46.385Z29/30 Read block 417 len 4096
120412024-02-15T03:02:46.385Z30/30 Write block 440 len 36864 data: 9 10 12 11 9 11 11 10 9
120422024-02-15T03:02:46.385Z----------------------------------------------------------------
120432024-02-15T03:02:46.385Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
120442024-02-15T03:02:46.386ZStates: Active Active Active
120452024-02-15T03:02:46.386ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
120462024-02-15T03:02:46.386Z 1 Acked 1000 Write 8 Done Sent Done false
120472024-02-15T03:02:46.386Z 2 Acked 1001 Read 8 Done Sent Done false
120482024-02-15T03:02:46.386Z 3 Acked 1002 Read 10 Done Sent Done false
120492024-02-15T03:02:46.386Z 4 Acked 1003 Read 1 Done Sent Done false
120502024-02-15T03:02:46.386Z 5 Acked 1004 Read 6 Done Sent Done false
120512024-02-15T03:02:46.386Z 6 Acked 1005 Read 1 Done Sent Done false
120522024-02-15T03:02:46.386Z 7 Acked 1006 Read 6 Done Sent Done false
120532024-02-15T03:02:46.386Z 8 Acked 1007 Read 8 Done Sent Done false
120542024-02-15T03:02:46.386Z 9 Acked 1008 Read 10 Done Sent Done false
120552024-02-15T03:02:46.386Z 10 Acked 1009 Flush 0 Done Sent Done false
120562024-02-15T03:02:46.386Z 11 Acked 1010 Write 10 Done Sent Done false
120572024-02-15T03:02:46.386Z 12 Acked 1011 Write 2 Done Sent Done false
120582024-02-15T03:02:46.387Z 13 Acked 1012 Write 2 Done Sent Done false
120592024-02-15T03:02:46.387Z 14 Acked 1013 Write 2 Done Sent Done false
120602024-02-15T03:02:46.387Z 15 Acked 1014 Read 5 Done Sent Done false
120612024-02-15T03:02:46.387Z 16 Acked 1015 Read 7 Done Sent Done false
120622024-02-15T03:02:46.387Z 17 Acked 1016 Write 9 Done Sent Done false
120632024-02-15T03:02:46.387Z 18 Acked 1017 Read 9 Done Sent Done false
120642024-02-15T03:02:46.387Z 19 Acked 1018 Read 7 Done Sent Done false
120652024-02-15T03:02:46.387Z 20 Acked 1019 Flush 0 Done Sent Done false
120662024-02-15T03:02:46.387Z 21 Acked 1020 Write 8 Done Sent Done false
120672024-02-15T03:02:46.387Z 22 Acked 1021 Read 8 Done Sent Done false
120682024-02-15T03:02:46.387Z 23 Acked 1022 Write 6 Done Sent Done false
120692024-02-15T03:02:46.387Z 24 Acked 1023 Read 6 Done Sent Done false
120702024-02-15T03:02:46.388Z 25 Acked 1024 Read 7 Done Sent Done false
120712024-02-15T03:02:46.388Z 26 Acked 1025 Write 8 Done Sent Done false
120722024-02-15T03:02:46.388Z 27 Acked 1026 Read 10 Done Sent Done false
120732024-02-15T03:02:46.388Z 28 Acked 1027 Read 9 Done Sent Done false
120742024-02-15T03:02:46.388Z 29 Acked 1028 Read 1 Done Sent Done false
120752024-02-15T03:02:46.388Z 30 Acked 1029 Write 9 Sent Sent Sent false
120762024-02-15T03:02:46.388Z STATES DS:0 DS:1 DS:2 TOTAL
120772024-02-15T03:02:46.388Z New 0 0 0 0
120782024-02-15T03:02:46.388Z Sent 1 30 1 32
120792024-02-15T03:02:46.388Z Done 29 0 29 58
120802024-02-15T03:02:46.388Z Skipped 0 0 0 0
120812024-02-15T03:02:46.388Z Error 0 0 0 0
120822024-02-15T03:02:46.388ZLast Flush: 1019 0 1019
120832024-02-15T03:02:46.389ZDownstairs last five completed:
120842024-02-15T03:02:46.389ZUpstairs last five completed: 30 29 28 27 26
120852024-02-15T03:02:46.389Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:45.296253523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1570}
120862024-02-15T03:02:46.389ZWrote out file "/tmp/test_repair_verify.data"
120872024-02-15T03:02:46.392Z
120882024-02-15T03:02:46.392ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
120892024-02-15T03:02:46.409ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120902024-02-15T03:02:46.410Z 0 000-019 94 94 94 160 160 160 F F F
120912024-02-15T03:02:46.410Z 1 020-039 101 101 101 169 169 169 F F F
120922024-02-15T03:02:46.410Z 2 040-059 98 98 98 165 165 165 F F F
120932024-02-15T03:02:46.410Z 3 060-079 108 98 108 184 165 184 F F F <---
120942024-02-15T03:02:46.410Z 4 080-099 108 104 108 184 175 184 F F F <---
120952024-02-15T03:02:46.410Z 5 100-119 105 105 105 179 179 179 F F F
120962024-02-15T03:02:46.410Z 6 120-139 99 99 99 167 167 167 F F F
120972024-02-15T03:02:46.413Z 7 140-159 106 106 106 180 180 180 F F F
120982024-02-15T03:02:46.414Z 8 160-179 100 100 100 168 168 168 F F F
120992024-02-15T03:02:46.414Z 9 180-199 102 102 102 173 173 173 F F F
121002024-02-15T03:02:46.414Z 10 200-219 104 104 104 178 178 178 F F F
121012024-02-15T03:02:46.414Z 11 220-239 104 104 104 178 178 178 F F F
121022024-02-15T03:02:46.414Z 12 240-259 102 102 102 170 170 170 T F T <---
121032024-02-15T03:02:46.414Z 13 260-279 108 106 108 184 180 184 F F F <---
121042024-02-15T03:02:46.414Z 14 280-299 108 106 108 184 181 184 F F F <---
121052024-02-15T03:02:46.414Z 15 300-319 99 99 99 167 167 167 F F F
121062024-02-15T03:02:46.414Z 16 320-339 108 106 108 184 181 184 F F F <---
121072024-02-15T03:02:46.414Z 17 340-359 105 105 105 179 179 179 F F F
121082024-02-15T03:02:46.414Z 18 360-379 105 105 105 179 179 179 F F F
121092024-02-15T03:02:46.414Z 19 380-399 106 106 106 181 181 181 F F F
121102024-02-15T03:02:46.415Z 20 400-419 108 104 108 184 177 184 F F F <---
121112024-02-15T03:02:46.415Z 21 420-439 104 104 104 176 176 176 F F F
121122024-02-15T03:02:46.415Z 22 440-459 106 106 106 181 181 181 T F T <---
121132024-02-15T03:02:46.415Z 23 460-479 101 101 101 169 169 169 T F T <---
121142024-02-15T03:02:46.415Z 24 480-499 108 106 108 184 181 184 F F F <---
121152024-02-15T03:02:46.415Z 25 500-519 105 105 105 179 179 179 F F F
121162024-02-15T03:02:46.415Z 26 520-539 106 106 106 181 181 181 F F F
121172024-02-15T03:02:46.415Z 27 540-559 107 107 107 182 182 182 F F F
121182024-02-15T03:02:46.415Z 28 560-579 107 107 107 182 182 182 T F T <---
121192024-02-15T03:02:46.415Z 29 580-599 106 106 106 181 181 181 F F F
121202024-02-15T03:02:46.415ZMax gen: 108, Max flush: 184
121212024-02-15T03:02:46.415ZError: Difference in extent metadata found!
121222024-02-15T03:02:46.415ZA Difference in extent metadata is expected here
121232024-02-15T03:02:46.415ZOn loop 54
121242024-02-15T03:02:46.416Z
121252024-02-15T03:02:46.421ZVerifying data now
121262024-02-15T03:02:50.994ZNo change after verify
121272024-02-15T03:02:50.994ZLoop: 54 Downstairs dump after verify (and repair):
121282024-02-15T03:02:51.010ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121292024-02-15T03:02:51.010Z 0 000-019 94 94 94 160 160 160 F F F
121302024-02-15T03:02:51.010Z 1 020-039 101 101 101 169 169 169 F F F
121312024-02-15T03:02:51.010Z 2 040-059 98 98 98 165 165 165 F F F
121322024-02-15T03:02:51.010Z 3 060-079 108 108 108 184 184 184 F F F
121332024-02-15T03:02:51.010Z 4 080-099 108 108 108 184 184 184 F F F
121342024-02-15T03:02:51.010Z 5 100-119 105 105 105 179 179 179 F F F
121352024-02-15T03:02:51.011Z 6 120-139 99 99 99 167 167 167 F F F
121362024-02-15T03:02:51.011Z 7 140-159 106 106 106 180 180 180 F F F
121372024-02-15T03:02:51.011Z 8 160-179 100 100 100 168 168 168 F F F
121382024-02-15T03:02:51.011Z 9 180-199 102 102 102 173 173 173 F F F
121392024-02-15T03:02:51.011Z 10 200-219 104 104 104 178 178 178 F F F
121402024-02-15T03:02:51.011Z 11 220-239 104 104 104 178 178 178 F F F
121412024-02-15T03:02:51.011Z 12 240-259 109 109 109 185 185 185 F F F
121422024-02-15T03:02:51.011Z 13 260-279 108 108 108 184 184 184 F F F
121432024-02-15T03:02:51.011Z 14 280-299 108 108 108 184 184 184 F F F
121442024-02-15T03:02:51.016Z 15 300-319 99 99 99 167 167 167 F F F
121452024-02-15T03:02:51.016Z 16 320-339 108 108 108 184 184 184 F F F
121462024-02-15T03:02:51.016Z 17 340-359 105 105 105 179 179 179 F F F
121472024-02-15T03:02:51.016Z 18 360-379 105 105 105 179 179 179 F F F
121482024-02-15T03:02:51.016Z 19 380-399 106 106 106 181 181 181 F F F
121492024-02-15T03:02:51.016Z 20 400-419 108 108 108 184 184 184 F F F
121502024-02-15T03:02:51.016Z 21 420-439 104 104 104 176 176 176 F F F
121512024-02-15T03:02:51.016Z 22 440-459 109 109 109 185 185 185 F F F
121522024-02-15T03:02:51.016Z 23 460-479 109 109 109 185 185 185 F F F
121532024-02-15T03:02:51.016Z 24 480-499 108 108 108 184 184 184 F F F
121542024-02-15T03:02:51.016Z 25 500-519 105 105 105 179 179 179 F F F
121552024-02-15T03:02:51.016Z 26 520-539 106 106 106 181 181 181 F F F
121562024-02-15T03:02:51.016Z 27 540-559 107 107 107 182 182 182 F F F
121572024-02-15T03:02:51.016Z 28 560-579 109 109 109 185 185 185 F F F
121582024-02-15T03:02:51.016Z 29 580-599 106 106 106 181 181 181 F F F
121592024-02-15T03:02:51.016ZMax gen: 109, Max flush: 185
121602024-02-15T03:02:51.016Z
121612024-02-15T03:02:51.016ZBegin loop 55
121622024-02-15T03:02:51.016ZDownstairs to restart: 0
121632024-02-15T03:02:51.036Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.944940856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578}
121642024-02-15T03:02:51.036Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: Crucible runtime is spawned
121652024-02-15T03:02:51.036Z\"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945327228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578}
121662024-02-15T03:02:51.036Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945358974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578}
121672024-02-15T03:02:51.050Z{"msg":"Crucible 3732cdd5-5f7e-47c5-a6bd-3ebbcb985b95 has session id: 50e0339d-15f0-479e-95ac-f163a73e1d94","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945383241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121682024-02-15T03:02:51.050Z{"msg":"Upstairs opts: Upstairs UUID: 3732cdd5-5f7e-47c5-a6bd-3ebbcb985b95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945428255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121692024-02-15T03:02:51.050Z{"msg":"Crucible stats registered with UUID: 3732cdd5-5f7e-47c5-a6bd-3ebbcb985b95","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945447273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121702024-02-15T03:02:51.050Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945484328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578}
121712024-02-15T03:02:51.050Z{"msg":"3732cdd5-5f7e-47c5-a6bd-3ebbcb985b95 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.94554676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121722024-02-15T03:02:51.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.94562828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121732024-02-15T03:02:51.050Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945682213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121742024-02-15T03:02:51.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945732367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121752024-02-15T03:02:51.050Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945760254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121762024-02-15T03:02:51.050Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945795459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121772024-02-15T03:02:51.051Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945822566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121782024-02-15T03:02:51.051Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945858771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121792024-02-15T03:02:51.051Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945888067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121802024-02-15T03:02:51.051Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945913934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121812024-02-15T03:02:51.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945940941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121822024-02-15T03:02:51.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.945980296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121832024-02-15T03:02:51.051Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:49.946044848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121842024-02-15T03:02:51.051Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:49.946082333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121852024-02-15T03:02:51.051Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:49.946205158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121862024-02-15T03:02:51.051Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.946240913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121872024-02-15T03:02:51.051Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.946256861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121882024-02-15T03:02:51.051Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.946364198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121892024-02-15T03:02:51.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.946624935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121902024-02-15T03:02:51.051Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.946673309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121912024-02-15T03:02:51.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.946725673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121922024-02-15T03:02:51.051Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.946771677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121932024-02-15T03:02:51.051Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.947239638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121942024-02-15T03:02:51.051Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.947275664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121952024-02-15T03:02:51.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.947849482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121962024-02-15T03:02:51.051Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.947887057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121972024-02-15T03:02:51.051Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.947902705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121982024-02-15T03:02:51.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.947923412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
121992024-02-15T03:02:51.051Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.94793793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122002024-02-15T03:02:51.052Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:49.947958438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122012024-02-15T03:02:52.039Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.94781208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122022024-02-15T03:02:52.050Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.948150007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122032024-02-15T03:02:52.050Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.948953277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122042024-02-15T03:02:52.050Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.948991022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122052024-02-15T03:02:52.051Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.949587267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122062024-02-15T03:02:52.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950160775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122072024-02-15T03:02:52.051Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950245784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122082024-02-15T03:02:52.051Z{"msg":"[0]R flush_numbers[0..12]: [160, 169, 165, 184, 184, 179, 167, 180, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.95028473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122092024-02-15T03:02:52.052Z{"msg":"[0]R generation[0..12]: [94, 101, 98, 108, 108, 105, 99, 106, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950303967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122102024-02-15T03:02:52.052Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950330114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122112024-02-15T03:02:52.052Z{"msg":"[1]R flush_numbers[0..12]: [160, 169, 165, 184, 184, 179, 167, 180, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950348802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578Wait for a query_work_queue command to finish before sending IO
122122024-02-15T03:02:52.052Z,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122132024-02-15T03:02:52.052Z{"msg":"[1]R generation[0..12]: [94, 101, 98, 108, 108, 105, 99, 106, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950382687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122142024-02-15T03:02:52.053Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950402175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122152024-02-15T03:02:52.053Z{"msg":"[2]R flush_numbers[0..12]: [160, 169, 165, 184, 184, 179, 167, 180, 168, 173, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950429252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122162024-02-15T03:02:52.053Z{"msg":"[2]R generation[0..12]: [94, 101, 98, 108, 108, 105, 99, 106, 100, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950448099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122172024-02-15T03:02:52.053Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950474626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122182024-02-15T03:02:52.053Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950491594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122192024-02-15T03:02:52.054Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.95052389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122202024-02-15T03:02:52.054Z{"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950545117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122212024-02-15T03:02:52.054Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950567364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122222024-02-15T03:02:52.054Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950584372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122232024-02-15T03:02:52.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950609889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122242024-02-15T03:02:52.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950627277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122252024-02-15T03:02:52.055Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950652704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122262024-02-15T03:02:52.055Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950671051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122272024-02-15T03:02:52.056Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950687699Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122282024-02-15T03:02:52.056Z{"msg":"3732cdd5-5f7e-47c5-a6bd-3ebbcb985b95 is now active with session: 50e0339d-15f0-479e-95ac-f163a73e1d94","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950703917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122292024-02-15T03:02:52.056Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950727344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578,"session_id":"50e0339d-15f0-479e-95ac-f163a73e1d94"}
122302024-02-15T03:02:52.056Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:50.950743152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578}
122312024-02-15T03:02:52.057ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122322024-02-15T03:02:52.057ZLoading write count information from file "/tmp/test_repair_verify.data"
122332024-02-15T03:02:52.058ZRun Repair workload
122342024-02-15T03:02:52.058Z01/30 Write block 238 len 40960 data: 10 10 10 10 10 8 8 9 9 9
122352024-02-15T03:02:52.058Z02/30 Read block 145 len 32768
122362024-02-15T03:02:52.058Z03/30 Read block 408 len 16384
122372024-02-15T03:02:52.058Z04/30 Write block 21 len 16384 data: 4 3 2 4
122382024-02-15T03:02:52.059Z05/30 Write block 136 len 24576 data: 9 8 9 8 10 11
122392024-02-15T03:02:52.080Z06/30 Write block 162 len 40960 data: 13 12 10 12 13 11 10 9 7 7
122402024-02-15T03:02:52.081Z07/30 Flush
122412024-02-15T03:02:52.081Z08/30 Write block 40 len 24576 data: 5 6 7 6 9 9
122422024-02-15T03:02:52.081Z09/30 Read block 128 len 24576
122432024-02-15T03:02:52.081Z10/30 Read block 583 len 8192
122442024-02-15T03:02:52.081Z11/30 Write block 199 len 8192 data: 9 9
122452024-02-15T03:02:52.081Z12/30 Read block 446 len 12288
122462024-02-15T03:02:52.081Z13/30 Write block 446 len 4096 data: 12
122472024-02-15T03:02:52.081Z14/30 Flush
122482024-02-15T03:02:52.082Z15/30 Write block 443 len 4096 data: 12
122492024-02-15T03:02:52.082Z16/30 Read block 146 len 12288
122502024-02-15T03:02:52.082Z17/30 Read block 241 len 32768
122512024-02-15T03:02:52.082Z18/30 Read block 127 len 8192
122522024-02-15T03:02:52.082Z19/30 Read block 48 len 32768
122532024-02-15T03:02:52.082Z20/30 Write block 575 len 24576 data: 14 16 16 13 11 12
122542024-02-15T03:02:52.082Z21/30 Read block 336 len 32768
122552024-02-15T03:02:52.083Z22/30 Read block 435 len 36864
122562024-02-15T03:02:52.083Z23/30 Read block 462 len 36864
122572024-02-15T03:02:52.083Z24/30 Read block 247 len 24576
122582024-02-15T03:02:52.091Z25/30 Read block 524 len 40960
122592024-02-15T03:02:52.092Z26/30 Flush
122602024-02-15T03:02:52.092Z27/30 Write block 244 len 8192 data: 9 10
122612024-02-15T03:02:52.092Z28/30 Write block 447 len 8192 data: 11 10
122622024-02-15T03:02:52.092Z29/30 Write block 548 len 28672 data: 7 6 5 5 5 8 10
122632024-02-15T03:02:52.092Z30/30 Write block 200 len 4096 data: 10
122642024-02-15T03:02:52.092Z----------------------------------------------------------------
122652024-02-15T03:02:52.093Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
122662024-02-15T03:02:52.093ZStates: Active Active Active
122672024-02-15T03:02:52.093ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122682024-02-15T03:02:52.093Z 1 Acked 1000 Write 10 Done Done Done false
122692024-02-15T03:02:52.093Z 2 Acked 1001 Read 8 Done Done Done false
122702024-02-15T03:02:52.093Z 3 Acked 1002 Read 4 Done Done Done false
122712024-02-15T03:02:52.093Z 4 Acked 1003 Write 4 Done Done Done false
122722024-02-15T03:02:52.093Z 5 Acked 1004 Write 6 Done Done Done false
122732024-02-15T03:02:52.093Z 6 Acked 1005 Write 10 Sent Done Done false
122742024-02-15T03:02:52.094Z 7 Acked 1006 Flush 0 Sent Done Done false
122752024-02-15T03:02:52.094Z 8 Acked 1007 Write 6 Sent Done Done false
122762024-02-15T03:02:52.094Z 9 Acked 1008 Read 6 Sent Done Done false
122772024-02-15T03:02:52.094Z 10 Acked 1009 Read 2 Sent Done Done false
122782024-02-15T03:02:52.094Z 11 Acked 1010 Write 2 Sent Done Done false
122792024-02-15T03:02:52.094Z 12 Acked 1011 Read 3 Sent Done Done false
122802024-02-15T03:02:52.094Z 13 Acked 1012 Write 1 Sent Done Done false
122812024-02-15T03:02:52.094Z 14 Acked 1013 Flush 0 Sent Done Done false
122822024-02-15T03:02:52.095Z 15 Acked 1014 Write 1 Sent Done Done false
122832024-02-15T03:02:52.095Z 16 Acked 1015 Read 3 Sent Done Done false
122842024-02-15T03:02:52.095Z 17 Acked 1016 Read 8 Sent Done Done false
122852024-02-15T03:02:52.095Z 18 Acked 1017 Read 2 Sent Done Done false
122862024-02-15T03:02:52.095Z 19 Acked 1018 Read 8 Sent Done Done false
122872024-02-15T03:02:52.095Z 20 Acked 1019 Write 6 Sent Done Done false
122882024-02-15T03:02:52.095Z 21 Acked 1020 Read 8 Sent Done Done false
122892024-02-15T03:02:52.096Z 22 Acked 1021 Read 9 Sent Done Done false
122902024-02-15T03:02:52.096Z 23 Acked 1022 Read 9 Sent Done Done false
122912024-02-15T03:02:52.096Z 24 Acked 1023 Read 6 Sent Done Done false
122922024-02-15T03:02:52.096Z 25 Acked 1024 Read 10 Sent Done Done false
122932024-02-15T03:02:52.096Z 26 Acked 1025 Flush 0 Sent Done Done false
122942024-02-15T03:02:52.096Z 27 Acked 1026 Write 2 Sent Done Done false
122952024-02-15T03:02:52.096Z 28 Acked 1027 Write 2 Sent Done Done false
122962024-02-15T03:02:52.096Z 29 Acked 1028 Write 7 Sent Sent Sent false
122972024-02-15T03:02:52.096Z 30 Acked 1029 Write 1 Sent Sent Sent false
122982024-02-15T03:02:52.096Z STATES DS:0 DS:1 DS:2 TOTAL
122992024-02-15T03:02:52.097Z New 0 0 0 0
123002024-02-15T03:02:52.097Z Sent 25 2 2 29
123012024-02-15T03:02:52.097Z Done 5 28 28 61
123022024-02-15T03:02:52.097Z Skipped 0 0 0 0
123032024-02-15T03:02:52.097Z Error 0 0 0 0
123042024-02-15T03:02:52.097ZLast Flush: 0 1025 1025
123052024-02-15T03:02:52.097ZDownstairs last five completed:
123062024-02-15T03:02:52.097ZUpstairs last five completed: 30 29 28 27 26
123072024-02-15T03:02:52.098Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:50.983136414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1578}
123082024-02-15T03:02:52.098ZWrote out file "/tmp/test_repair_verify.data"
123092024-02-15T03:02:52.098Z
123102024-02-15T03:02:52.098ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123112024-02-15T03:02:52.105ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123122024-02-15T03:02:52.105Z 0 000-019 94 94 94 160 160 160 F F F
123132024-02-15T03:02:52.105Z 1 020-039 101 110 110 169 186 186 T F F <---
123142024-02-15T03:02:52.105Z 2 040-059 98 110 110 165 187 187 F F F <---
123152024-02-15T03:02:52.105Z 3 060-079 108 108 108 184 184 184 F F F
123162024-02-15T03:02:52.106Z 4 080-099 108 108 108 184 184 184 F F F
123172024-02-15T03:02:52.106Z 5 100-119 105 105 105 179 179 179 F F F
123182024-02-15T03:02:52.106Z 6 120-139 99 110 110 167 186 186 T F F <---
123192024-02-15T03:02:52.106Z 7 140-159 106 110 110 180 186 186 T F F <---
123202024-02-15T03:02:52.106Z 8 160-179 100 110 110 168 186 186 F F F <---
123212024-02-15T03:02:52.106Z 9 180-199 102 110 110 173 187 187 F F F <---
123222024-02-15T03:02:52.107Z 10 200-219 104 110 110 178 187 187 F T T <---
123232024-02-15T03:02:52.107Z 11 220-239 104 110 110 178 186 186 T F F <---
123242024-02-15T03:02:52.107Z 12 240-259 109 110 110 185 186 186 T T T <---
123252024-02-15T03:02:52.107Z 13 260-279 108 108 108 184 184 184 F F F
123262024-02-15T03:02:52.107Z 14 280-299 108 108 108 184 184 184 F F F
123272024-02-15T03:02:52.107Z 15 300-319 99 99 99 167 167 167 F F F
123282024-02-15T03:02:52.108Z 16 320-339 108 108 108 184 184 184 F F F
123292024-02-15T03:02:52.108Z 17 340-359 105 105 105 179 179 179 F F F
123302024-02-15T03:02:52.108Z 18 360-379 105 105 105 179 179 179 F F F
123312024-02-15T03:02:52.108Z 19 380-399 106 106 106 181 181 181 F F F
123322024-02-15T03:02:52.108Z 20 400-419 108 108 108 184 184 184 F F F
123332024-02-15T03:02:52.108Z 21 420-439 104 104 104 176 176 176 F F F
123342024-02-15T03:02:52.109Z 22 440-459 109 110 110 185 188 188 F T T <---
123352024-02-15T03:02:52.109Z 23 460-479 109 109 109 185 185 185 F F F
123362024-02-15T03:02:52.109Z 24 480-499 108 108 108 184 184 184 F F F
123372024-02-15T03:02:52.109Z 25 500-519 105 105 105 179 179 179 F F F
123382024-02-15T03:02:52.109Z 26 520-539 106 106 106 181 181 181 F F F
123392024-02-15T03:02:52.110Z 27 540-559 107 107 107 182 182 182 F T T <---
123402024-02-15T03:02:52.110Z 28 560-579 109 110 110 185 188 188 F F F <---
123412024-02-15T03:02:52.110Z 29 580-599 106 110 110 181 188 188 F F F <---
123422024-02-15T03:02:52.110ZMax gen: 110, Max flush: 188
123432024-02-15T03:02:52.110ZError: Difference in extent metadata found!
123442024-02-15T03:02:52.173ZA Difference in extent metadata is expected here
123452024-02-15T03:02:52.173ZOn loop 55
123462024-02-15T03:02:52.173Z
123472024-02-15T03:02:52.173ZVerifying data now
123482024-02-15T03:02:57.766ZNo change after verify
123492024-02-15T03:02:57.767ZLoop: 55 Downstairs dump after verify (and repair):
123502024-02-15T03:02:57.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123512024-02-15T03:02:57.774Z 0 000-019 94 94 94 160 160 160 F F F
123522024-02-15T03:02:57.775Z 1 020-039 110 110 110 186 186 186 F F F
123532024-02-15T03:02:57.775Z 2 040-059 110 110 110 187 187 187 F F F
123542024-02-15T03:02:57.775Z 3 060-079 108 108 108 184 184 184 F F F
123552024-02-15T03:02:57.775Z 4 080-099 108 108 108 184 184 184 F F F
123562024-02-15T03:02:57.790Z 5 100-119 105 105 105 179 179 179 F F F
123572024-02-15T03:02:57.790Z 6 120-139 110 110 110 186 186 186 F F F
123582024-02-15T03:02:57.790Z 7 140-159 110 110 110 186 186 186 F F F
123592024-02-15T03:02:57.790Z 8 160-179 110 110 110 186 186 186 F F F
123602024-02-15T03:02:57.790Z 9 180-199 110 110 110 187 187 187 F F F
123612024-02-15T03:02:57.790Z 10 200-219 111 111 111 189 189 189 F F F
123622024-02-15T03:02:57.791Z 11 220-239 110 110 110 186 186 186 F F F
123632024-02-15T03:02:57.791Z 12 240-259 111 111 111 189 189 189 F F F
123642024-02-15T03:02:57.791Z 13 260-279 108 108 108 184 184 184 F F F
123652024-02-15T03:02:57.791Z 14 280-299 108 108 108 184 184 184 F F F
123662024-02-15T03:02:57.791Z 15 300-319 99 99 99 167 167 167 F F F
123672024-02-15T03:02:57.791Z 16 320-339 108 108 108 184 184 184 F F F
123682024-02-15T03:02:57.791Z 17 340-359 105 105 105 179 179 179 F F F
123692024-02-15T03:02:57.791Z 18 360-379 105 105 105 179 179 179 F F F
123702024-02-15T03:02:57.791Z 19 380-399 106 106 106 181 181 181 F F F
123712024-02-15T03:02:57.791Z 20 400-419 108 108 108 184 184 184 F F F
123722024-02-15T03:02:57.791Z 21 420-439 104 104 104 176 176 176 F F F
123732024-02-15T03:02:57.792Z 22 440-459 111 111 111 189 189 189 F F F
123742024-02-15T03:02:57.792Z 23 460-479 109 109 109 185 185 185 F F F
123752024-02-15T03:02:57.792Z 24 480-499 108 108 108 184 184 184 F F F
123762024-02-15T03:02:57.792Z 25 500-519 105 105 105 179 179 179 F F F
123772024-02-15T03:02:57.792Z 26 520-539 106 106 106 181 181 181 F F F
123782024-02-15T03:02:57.792Z 27 540-559 111 111 111 189 189 189 F F F
123792024-02-15T03:02:57.792Z 28 560-579 110 110 110 188 188 188 F F F
123802024-02-15T03:02:57.792Z 29 580-599 110 110 110 188 188 188 F F F
123812024-02-15T03:02:57.792ZMax gen: 111, Max flush: 189
123822024-02-15T03:02:57.793Z
123832024-02-15T03:02:57.793ZBegin loop 56
123842024-02-15T03:02:57.793ZDownstairs to restart: 1
123852024-02-15T03:02:57.822Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.704574784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586}
123862024-02-15T03:02:57.822ZCrucible runtime is spawned
123872024-02-15T03:02:57.822Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.704979493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586}
123882024-02-15T03:02:57.822Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705018319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586}
123892024-02-15T03:02:57.823Z{"msg":"Crucible 1cfa2c24-f91a-4836-af4e-986d9b786a28 has session id: eb0323e5-7cc2-4db2-b9ab-345d1737a616","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705042696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123902024-02-15T03:02:57.823Z{"msg":"Upstairs opts: Upstairs UUID: 1cfa2c24-f91a-4836-af4e-986d9b786a28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.70508753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123912024-02-15T03:02:57.823Z{"msg":"Crucible stats registered with UUID: 1cfa2c24-f91a-4836-af4e-986d9b786a28","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705123775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123922024-02-15T03:02:57.823Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705153832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586}
123932024-02-15T03:02:57.823Z{"msg":"1cfa2c24-f91a-4836-af4e-986d9b786a28 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705188757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123942024-02-15T03:02:57.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705210704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123952024-02-15T03:02:57.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705233872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123962024-02-15T03:02:57.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705274676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123972024-02-15T03:02:57.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.70532627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123982024-02-15T03:02:57.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705360096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
123992024-02-15T03:02:57.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705383153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124002024-02-15T03:02:57.823Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705494149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124012024-02-15T03:02:57.823Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.70556294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124022024-02-15T03:02:57.823Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705588887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124032024-02-15T03:02:57.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705633662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124042024-02-15T03:02:57.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705661318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124052024-02-15T03:02:57.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.705751367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124062024-02-15T03:02:57.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.706331874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124072024-02-15T03:02:57.824Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.706369299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124082024-02-15T03:02:57.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.706387007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124092024-02-15T03:02:57.824Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.70644444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124102024-02-15T03:02:57.824Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.706996721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124112024-02-15T03:02:57.824Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.707045255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124122024-02-15T03:02:57.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.707355746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124132024-02-15T03:02:57.824Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.707391661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124142024-02-15T03:02:57.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.7076408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124152024-02-15T03:02:57.824Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.70772032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124162024-02-15T03:02:57.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.707738428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124172024-02-15T03:02:57.825Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.707766244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124182024-02-15T03:02:57.825Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.707781922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124192024-02-15T03:02:57.825Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.707808519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124202024-02-15T03:02:57.825Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708001525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124212024-02-15T03:02:57.825Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708467396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124222024-02-15T03:02:57.825Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708523409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124232024-02-15T03:02:57.825Z{"msg":"[0]R flush_numbers[0..12]: [160, 186, 187, 184, 184, 179, 186, 186, 186, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708558595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124242024-02-15T03:02:57.825Z{"msg":"[0]R generation[0..12]: [94, 110, 110, 108, 108, 105, 110, 110, 110, 110, 111, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708586032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124252024-02-15T03:02:57.825Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708626057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124262024-02-15T03:02:57.825Z{"msg":"[1]R flush_numbers[0..12]: [160, 186, 187, 184, 184, 179, 186, 186, 186, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708653493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124272024-02-15T03:02:57.825ZWait for a query_work_queue command to finish before sending IO
124282024-02-15T03:02:57.825Z{"msg":"[1]R generation[0..12]: [94, 110, 110, 108, 108, 105, 110, 110, 110, 110, 111, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708683509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124292024-02-15T03:02:57.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708709316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124302024-02-15T03:02:57.825Z{"msg":"[2]R flush_numbers[0..12]: [160, 186, 187, 184, 184, 179, 186, 186, 186, 187, 189, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708736193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124312024-02-15T03:02:57.826Z{"msg":"[2]R generation[0..12]: [94, 110, 110, 108, 108, 105, 110, 110, 110, 110, 111, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708769479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124322024-02-15T03:02:57.826Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708796495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124332024-02-15T03:02:57.826Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708824592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124342024-02-15T03:02:57.826Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708849019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124352024-02-15T03:02:57.826Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708875076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124362024-02-15T03:02:57.826Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708898762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124372024-02-15T03:02:57.826Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.70892149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124382024-02-15T03:02:57.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708944397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124392024-02-15T03:02:57.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.708972863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124402024-02-15T03:02:57.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.70899759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124412024-02-15T03:02:57.826Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124422024-02-15T03:02:57.826Z,"time":"2024-02-15T03:02:56.709052423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124432024-02-15T03:02:57.826Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.709084919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124442024-02-15T03:02:57.827Z{"msg":"1cfa2c24-f91a-4836-af4e-986d9b786a28 is now active with session: eb0323e5-7cc2-4db2-b9ab-345d1737a616","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.709111816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124452024-02-15T03:02:57.827Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.709125704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586,"session_id":"eb0323e5-7cc2-4db2-b9ab-345d1737a616"}
124462024-02-15T03:02:57.827Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:02:56.709146442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586}
124472024-02-15T03:02:57.847ZLoading write count information from file "/tmp/test_repair_verify.data"
124482024-02-15T03:02:57.847ZRun Repair workload
124492024-02-15T03:02:57.847Z01/30 Write block 142 len 4096 data: 10
124502024-02-15T03:02:57.847Z02/30 Read block 355 len 20480
124512024-02-15T03:02:57.848Z03/30 Write block 228 len 24576 data: 10 8 7 10 12 12
124522024-02-15T03:02:57.848Z04/30 Write block 59 len 36864 data: 8 8 8 9 9 9 6 5 5
124532024-02-15T03:02:57.848Z05/30 Read block 323 len 16384
124542024-02-15T03:02:57.848Z06/30 Flush
124552024-02-15T03:02:57.848Z07/30 Write block 413 len 12288 data: 16 13 10
124562024-02-15T03:02:57.848Z08/30 Read block 77 len 16384
124572024-02-15T03:02:57.848Z09/30 Read block 274 len 28672
124582024-02-15T03:02:57.848Z10/30 Write block 104 len 36864 data: 10 9 7 6 5 5 4 5 5
124592024-02-15T03:02:57.849Z11/30 Read block 192 len 8192
124602024-02-15T03:02:57.849Z12/30 Read block 288 len 40960
124612024-02-15T03:02:57.849Z13/30 Read block 426 len 24576
124622024-02-15T03:02:57.849Z14/30 Flush
124632024-02-15T03:02:57.849Z15/30 Write block 490 len 36864 data: 11 10 11 9 10 11 11 11 11
124642024-02-15T03:02:57.849Z16/30 Read block 174 len 28672
124652024-02-15T03:02:57.849Z17/30 Write block 131 len 12288 data: 6 8 10
124662024-02-15T03:02:57.849Z18/30 Read block 68 len 12288
124672024-02-15T03:02:57.849Z19/30 Write block 585 len 4096 data: 10
124682024-02-15T03:02:57.849Z20/30 Write block 50 len 36864 data: 8 6 8 9 9 11 9 11 11
124692024-02-15T03:02:57.849Z21/30 Read block 99 len 24576
124702024-02-15T03:02:57.849Z22/30 Write block 458 len 36864 data: 7 7 10 10 9 6 6 7 9
124712024-02-15T03:02:57.850Z23/30 Read block 114 len 20480
124722024-02-15T03:02:57.850Z24/30 Read block 438 len 20480
124732024-02-15T03:02:57.850Z25/30 Write block 93 len 32768 data: 5 8 7 10 12 13 11 10
124742024-02-15T03:02:57.850Z26/30 Write block 134 len 32768 data: 9 9 10 9 10 9 11 12
124752024-02-15T03:02:57.850Z27/30 Read block 267 len 32768
124762024-02-15T03:02:57.850Z28/30 Write block 376 len 12288 data: 9 9 10
124772024-02-15T03:02:57.850Z29/30 Read block 430 len 16384
124782024-02-15T03:02:57.850Z30/30 Read block 283 len 16384
124792024-02-15T03:02:57.850Z----------------------------------------------------------------
124802024-02-15T03:02:57.850Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
124812024-02-15T03:02:57.850ZStates: Active Active Active
124822024-02-15T03:02:57.850ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
124832024-02-15T03:02:57.850Z 1 Acked 1000 Write 1 Done Sent Done false
124842024-02-15T03:02:57.851Z 2 Acked 1001 Read 5 Done Sent Done false
124852024-02-15T03:02:57.851Z 3 Acked 1002 Write 6 Done Sent Done false
124862024-02-15T03:02:57.851Z 4 Acked 1003 Write 9 Done Sent Done false
124872024-02-15T03:02:57.851Z 5 Acked 1004 Read 4 Done Sent Done false
124882024-02-15T03:02:57.851Z 6 Acked 1005 Flush 0 Done Sent Done false
124892024-02-15T03:02:57.851Z 7 Acked 1006 Write 3 Done Sent Done false
124902024-02-15T03:02:57.851Z 8 Acked 1007 Read 4 Done Sent Done false
124912024-02-15T03:02:57.851Z 9 Acked 1008 Read 7 Done Sent Done false
124922024-02-15T03:02:57.851Z 10 Acked 1009 Write 9 Done Sent Done false
124932024-02-15T03:02:57.851Z 11 Acked 1010 Read 2 Done Sent Done false
124942024-02-15T03:02:57.851Z 12 Acked 1011 Read 10 Done Sent Done false
124952024-02-15T03:02:57.851Z 13 Acked 1012 Read 6 Done Sent Done false
124962024-02-15T03:02:57.851Z 14 Acked 1013 Flush 0 Done Sent Done false
124972024-02-15T03:02:57.852Z 15 Acked 1014 Write 9 Done Sent Done false
124982024-02-15T03:02:57.852Z 16 Acked 1015 Read 7 Done Sent Done false
124992024-02-15T03:02:57.852Z 17 Acked 1016 Write 3 Done Sent Done false
125002024-02-15T03:02:57.852Z 18 Acked 1017 Read 3 Done Sent Done false
125012024-02-15T03:02:57.852Z 19 Acked 1018 Write 1 Done Sent Done false
125022024-02-15T03:02:57.852Z 20 Acked 1019 Write 9 Done Sent Done false
125032024-02-15T03:02:57.852Z 21 Acked 1020 Read 6 Done Sent Done false
125042024-02-15T03:02:57.852Z 22 Acked 1021 Write 9 Done Sent Done false
125052024-02-15T03:02:57.852Z 23 Acked 1022 Read 5 Done Sent Done false
125062024-02-15T03:02:57.853Z 24 Acked 1023 Read 5 Done Sent Done false
125072024-02-15T03:02:57.853Z 25 Acked 1024 Write 8 Done Sent Done false
125082024-02-15T03:02:57.853Z 26 Acked 1025 Write 8 Done Sent Done false
125092024-02-15T03:02:57.853Z 27 Acked 1026 Read 8 Done Sent Done false
125102024-02-15T03:02:57.853Z 28 Acked 1027 Write 3 Done Sent Done false
125112024-02-15T03:02:57.853Z 29 Acked 1028 Read 4 Done Sent Done false
125122024-02-15T03:02:57.853Z 30 Acked 1029 Read 4 Done Sent Sent false
125132024-02-15T03:02:57.853Z STATES DS:0 DS:1 DS:2 TOTAL
125142024-02-15T03:02:57.853Z New 0 0 0 0
125152024-02-15T03:02:57.853Z Sent 0 30 1 31
125162024-02-15T03:02:57.853Z Done 30 0 29 59
125172024-02-15T03:02:57.853Z Skipped 0 0 0 0
125182024-02-15T03:02:57.853Z Error 0 0 0 0
125192024-02-15T03:02:57.854ZLast Flush: 1013 0 1013
125202024-02-15T03:02:57.854ZDownstairs last five completed:
125212024-02-15T03:02:57.854ZUpstairs last five completed: 30 29 28 27 26
125222024-02-15T03:02:57.854Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:02:56.740255114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1586}
125232024-02-15T03:02:57.854ZWrote out file "/tmp/test_repair_verify.data"
125242024-02-15T03:02:57.901Z
125252024-02-15T03:02:57.902ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125262024-02-15T03:02:57.902ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125272024-02-15T03:02:57.902Z 0 000-019 94 94 94 160 160 160 F F F
125282024-02-15T03:02:57.902Z 1 020-039 110 110 110 186 186 186 F F F
125292024-02-15T03:02:57.903Z 2 040-059 112 110 112 190 187 190 T F T <---
125302024-02-15T03:02:57.903Z 3 060-079 112 108 112 190 184 190 F F F <---
125312024-02-15T03:02:57.903Z 4 080-099 108 108 108 184 184 184 T F T <---
125322024-02-15T03:02:57.903Z 5 100-119 112 105 112 191 179 191 T F T <---
125332024-02-15T03:02:57.903Z 6 120-139 110 110 110 186 186 186 T F T <---
125342024-02-15T03:02:57.903Z 7 140-159 112 110 112 190 186 190 T F T <---
125352024-02-15T03:02:57.903Z 8 160-179 110 110 110 186 186 186 F F F
125362024-02-15T03:02:57.903Z 9 180-199 110 110 110 187 187 187 F F F
125372024-02-15T03:02:57.904Z 10 200-219 111 111 111 189 189 189 F F F
125382024-02-15T03:02:57.904Z 11 220-239 112 110 112 190 186 190 F F F <---
125392024-02-15T03:02:57.904Z 12 240-259 111 111 111 189 189 189 F F F
125402024-02-15T03:02:57.904Z 13 260-279 108 108 108 184 184 184 F F F
125412024-02-15T03:02:57.904Z 14 280-299 108 108 108 184 184 184 F F F
125422024-02-15T03:02:57.904Z 15 300-319 99 99 99 167 167 167 F F F
125432024-02-15T03:02:57.904Z 16 320-339 108 108 108 184 184 184 F F F
125442024-02-15T03:02:57.904Z 17 340-359 105 105 105 179 179 179 F F F
125452024-02-15T03:02:57.904Z 18 360-379 105 105 105 179 179 179 T F T <---
125462024-02-15T03:02:57.904Z 19 380-399 106 106 106 181 181 181 F F F
125472024-02-15T03:02:57.904Z 20 400-419 112 108 112 191 184 191 F F F <---
125482024-02-15T03:02:57.904Z 21 420-439 104 104 104 176 176 176 F F F
125492024-02-15T03:02:57.904Z 22 440-459 111 111 111 189 189 189 T F T <---
125502024-02-15T03:02:57.905Z 23 460-479 109 109 109 185 185 185 T F T <---
125512024-02-15T03:02:57.905Z 24 480-499 108 108 108 184 184 184 T F T <---
125522024-02-15T03:02:57.905Z 25 500-519 105 105 105 179 179 179 F F F
125532024-02-15T03:02:57.905Z 26 520-539 106 106 106 181 181 181 F F F
125542024-02-15T03:02:57.905Z 27 540-559 111 111 111 189 189 189 F F F
125552024-02-15T03:02:57.905Z 28 560-579 110 110 110 188 188 188 F F F
125562024-02-15T03:02:57.905Z 29 580-599 110 110 110 188 188 188 T F T <---
125572024-02-15T03:02:57.905ZMax gen: 112, Max flush: 191
125582024-02-15T03:02:57.905ZError: Difference in extent metadata found!
125592024-02-15T03:02:57.905ZA Difference in extent metadata is expected here
125602024-02-15T03:02:57.905ZOn loop 56
125612024-02-15T03:02:57.905Z
125622024-02-15T03:02:57.919ZVerifying data now
125632024-02-15T03:03:02.528ZNo change after verify
125642024-02-15T03:03:02.528ZLoop: 56 Downstairs dump after verify (and repair):
125652024-02-15T03:03:02.558ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125662024-02-15T03:03:02.558Z 0 000-019 94 94 94 160 160 160 F F F
125672024-02-15T03:03:02.559Z 1 020-039 110 110 110 186 186 186 F F F
125682024-02-15T03:03:02.559Z 2 040-059 113 113 113 192 192 192 F F F
125692024-02-15T03:03:02.559Z 3 060-079 112 112 112 190 190 190 F F F
125702024-02-15T03:03:02.559Z 4 080-099 113 113 113 192 192 192 F F F
125712024-02-15T03:03:02.559Z 5 100-119 113 113 113 192 192 192 F F F
125722024-02-15T03:03:02.564Z 6 120-139 113 113 113 192 192 192 F F F
125732024-02-15T03:03:02.564Z 7 140-159 113 113 113 192 192 192 F F F
125742024-02-15T03:03:02.564Z 8 160-179 110 110 110 186 186 186 F F F
125752024-02-15T03:03:02.564Z 9 180-199 110 110 110 187 187 187 F F F
125762024-02-15T03:03:02.564Z 10 200-219 111 111 111 189 189 189 F F F
125772024-02-15T03:03:02.564Z 11 220-239 112 112 112 190 190 190 F F F
125782024-02-15T03:03:02.565Z 12 240-259 111 111 111 189 189 189 F F F
125792024-02-15T03:03:02.565Z 13 260-279 108 108 108 184 184 184 F F F
125802024-02-15T03:03:02.565Z 14 280-299 108 108 108 184 184 184 F F F
125812024-02-15T03:03:02.565Z 15 300-319 99 99 99 167 167 167 F F F
125822024-02-15T03:03:02.565Z 16 320-339 108 108 108 184 184 184 F F F
125832024-02-15T03:03:02.565Z 17 340-359 105 105 105 179 179 179 F F F
125842024-02-15T03:03:02.565Z 18 360-379 113 113 113 192 192 192 F F F
125852024-02-15T03:03:02.565Z 19 380-399 106 106 106 181 181 181 F F F
125862024-02-15T03:03:02.566Z 20 400-419 112 112 112 191 191 191 F F F
125872024-02-15T03:03:02.566Z 21 420-439 104 104 104 176 176 176 F F F
125882024-02-15T03:03:02.566Z 22 440-459 113 113 113 192 192 192 F F F
125892024-02-15T03:03:02.566Z 23 460-479 113 113 113 192 192 192 F F F
125902024-02-15T03:03:02.566Z 24 480-499 113 113 113 192 192 192 F F F
125912024-02-15T03:03:02.566Z 25 500-519 105 105 105 179 179 179 F F F
125922024-02-15T03:03:02.567Z 26 520-539 106 106 106 181 181 181 F F F
125932024-02-15T03:03:02.567Z 27 540-559 111 111 111 189 189 189 F F F
125942024-02-15T03:03:02.567Z 28 560-579 110 110 110 188 188 188 F F F
125952024-02-15T03:03:02.567Z 29 580-599 113 113 113 192 192 192 F F F
125962024-02-15T03:03:02.567ZMax gen: 113, Max flush: 192
125972024-02-15T03:03:02.567Z
125982024-02-15T03:03:02.567ZBegin loop 57
125992024-02-15T03:03:02.568ZDownstairs to restart: 0
126002024-02-15T03:03:02.578Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476331808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594}
126012024-02-15T03:03:02.578Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476602204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594}
126022024-02-15T03:03:02.578Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476639149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594}
126032024-02-15T03:03:02.579Z{"msg":"Crucible 79613202-bc6b-4fea-83e6-933fbad78e20 has session id: 0b2d3442-dd58-4867-9fae-d2d68dc5bf36","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476662156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126042024-02-15T03:03:02.579Z{"msg":"Upstairs opts: Upstairs UUID: 79613202-bc6b-4fea-83e6-933fbad78e20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476706271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126052024-02-15T03:03:02.579ZCrucible runtime is spawned
126062024-02-15T03:03:02.579Z{"msg":"Crucible stats registered with UUID: 79613202-bc6b-4fea-83e6-933fbad78e20","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476739937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126072024-02-15T03:03:02.579Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476822596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594}
126082024-02-15T03:03:02.579Z{"msg":"79613202-bc6b-4fea-83e6-933fbad78e20 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476860602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126092024-02-15T03:03:02.580Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476892927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126102024-02-15T03:03:02.580Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476917544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126112024-02-15T03:03:02.580Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476939042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126122024-02-15T03:03:02.580Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.476979007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126132024-02-15T03:03:02.580Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.477046558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126142024-02-15T03:03:02.580Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.477083574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126152024-02-15T03:03:02.580Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.477108571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126162024-02-15T03:03:02.580Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.477141366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126172024-02-15T03:03:02.580Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.477172483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126182024-02-15T03:03:02.580Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.477297817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126192024-02-15T03:03:02.581Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.47735136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126202024-02-15T03:03:02.581Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.477380246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126212024-02-15T03:03:02.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.47806714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126222024-02-15T03:03:02.581Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.478105666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126232024-02-15T03:03:02.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.478123033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126242024-02-15T03:03:02.581Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.478155419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126252024-02-15T03:03:02.581Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.478729867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126262024-02-15T03:03:02.581Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.478891747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126272024-02-15T03:03:02.582Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479125558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126282024-02-15T03:03:02.582Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479160863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126292024-02-15T03:03:02.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479329152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126302024-02-15T03:03:02.582Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479375416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126312024-02-15T03:03:02.582Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479403213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126322024-02-15T03:03:02.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479434739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126332024-02-15T03:03:02.582Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479468915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126342024-02-15T03:03:02.583Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479541746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126352024-02-15T03:03:02.583Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.479766227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126362024-02-15T03:03:02.583Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480263485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126372024-02-15T03:03:02.583Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.48030803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126382024-02-15T03:03:02.583Z{"msg":"[0]R flush_numbers[0..12]: [160, 186, 192, 190, 192, 192, 192, 192, 186, 187, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480327437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126392024-02-15T03:03:02.583Z{"msg":"[0]R generation[0..12]: [94, 110, 113, 112, 113, 113, 113, 113, 110, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480350304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126402024-02-15T03:03:02.583Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480365442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126412024-02-15T03:03:02.583Z{"msg":"[1]R flush_numbers[0..12]: [160, 186, 192, 190, 192, 192, 192, 192, 186, 187, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.48038671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126422024-02-15T03:03:02.584Z{"msg":"[1]R generation[0..12]: [94, 110, 113, 112, 113, 113, 113, 113, 110, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480401048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126432024-02-15T03:03:02.584Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480416136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126442024-02-15T03:03:02.584Z{"msg":"[2]R flush_numbers[0..12]: [160, 186, 192, 190, 192, 192, 192, 192, 186, 187, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480443163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126452024-02-15T03:03:02.584Z{"msg":"[2]R generation[0..12]: [94, 110, 113, 112, 113, 113, 113, 113, 110, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.48046511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126462024-02-15T03:03:02.584Z{Wait for a query_work_queue command to finish before sending IO
126472024-02-15T03:03:02.584Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480499746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126482024-02-15T03:03:02.585Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480527222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126492024-02-15T03:03:02.585Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480548959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126502024-02-15T03:03:02.585Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480586995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126512024-02-15T03:03:02.585Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480617771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126522024-02-15T03:03:02.585Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480659535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126532024-02-15T03:03:02.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480679333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126542024-02-15T03:03:02.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480709529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126552024-02-15T03:03:02.585Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480732046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126562024-02-15T03:03:02.586Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480769222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126572024-02-15T03:03:02.586Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480794219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126582024-02-15T03:03:02.586Z{"msg":"79613202-bc6b-4fea-83e6-933fbad78e20 is now active with session: 0b2d3442-dd58-4867-9fae-d2d68dc5bf36","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480832474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126592024-02-15T03:03:02.586Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.48085957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594,"session_id":"0b2d3442-dd58-4867-9fae-d2d68dc5bf36"}
126602024-02-15T03:03:02.586Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:01.480873619Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594}
126612024-02-15T03:03:02.586ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
126622024-02-15T03:03:02.586ZLoading write count information from file "/tmp/test_repair_verify.data"
126632024-02-15T03:03:02.586ZRun Repair workload
126642024-02-15T03:03:02.586Z01/30 Write block 163 len 8192 data: 13 11
126652024-02-15T03:03:02.594Z02/30 Write block 465 len 24576 data: 8 10 9 9 10 11
126662024-02-15T03:03:02.594Z03/30 Read block 574 len 16384
126672024-02-15T03:03:02.594Z04/30 Flush
126682024-02-15T03:03:02.594Z05/30 Write block 94 len 16384 data: 9 8 11 13
126692024-02-15T03:03:02.594Z06/30 Read block 24 len 4096
126702024-02-15T03:03:02.594Z07/30 Read block 560 len 16384
126712024-02-15T03:03:02.594Z08/30 Write block 276 len 24576 data: 8 11 8 9 10 8
126722024-02-15T03:03:02.595Z09/30 Read block 175 len 4096
126732024-02-15T03:03:02.595Z10/30 Write block 82 len 12288 data: 7 6 8
126742024-02-15T03:03:02.595Z11/30 Read block 285 len 24576
126752024-02-15T03:03:02.595Z12/30 Read block 123 len 4096
126762024-02-15T03:03:02.595Z13/30 Write block 242 len 8192 data: 11 9
126772024-02-15T03:03:02.595Z14/30 Read block 96 len 8192
126782024-02-15T03:03:02.595Z15/30 Read block 62 len 24576
126792024-02-15T03:03:02.596Z16/30 Write block 91 len 8192 data: 6 5
126802024-02-15T03:03:02.596Z17/30 Write block 252 len 32768 data: 10 9 9 8 7 9 9 10
126812024-02-15T03:03:02.596Z18/30 Write block 242 len 12288 data: 12 10 10
126822024-02-15T03:03:02.596Z19/30 Write block 409 len 36864 data: 17 18 18 16 17 14 11 9 7
126832024-02-15T03:03:02.596Z20/30 Read block 202 len 36864
126842024-02-15T03:03:02.596Z21/30 Flush
126852024-02-15T03:03:02.601Z22/30 Flush
126862024-02-15T03:03:02.601Z23/30 Write block 86 len 28672 data: 10 10 8 6 7 7 6
126872024-02-15T03:03:02.601Z24/30 Read block 414 len 24576
126882024-02-15T03:03:02.602Z25/30 Write block 359 len 40960 data: 10 8 8 9 7 9 9 11 12 9
126892024-02-15T03:03:02.602Z26/30 Read block 220 len 4096
126902024-02-15T03:03:02.602Z27/30 Write block 334 len 4096 data: 15
126912024-02-15T03:03:02.602Z28/30 Write block 433 len 32768 data: 6 6 8 10 11 11 9 10
126922024-02-15T03:03:02.602Z29/30 Read block 81 len 28672
126932024-02-15T03:03:02.609Z30/30 Write block 297 len 32768 data: 8 8 9 8 11 11 13 13
126942024-02-15T03:03:02.609Z----------------------------------------------------------------
126952024-02-15T03:03:02.609Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
126962024-02-15T03:03:02.609ZStates: Active Active Active
126972024-02-15T03:03:02.610ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
126982024-02-15T03:03:02.610Z 1 Acked 1000 Write 2 Done Done Done false
126992024-02-15T03:03:02.610Z 2 Acked 1001 Write 6 Done Done Done false
127002024-02-15T03:03:02.610Z 3 Acked 1002 Read 4 Sent Done Done false
127012024-02-15T03:03:02.610Z 4 Acked 1003 Flush 0 Sent Done Done false
127022024-02-15T03:03:02.610Z 5 Acked 1004 Write 4 Sent Done Done false
127032024-02-15T03:03:02.611Z 6 Acked 1005 Read 1 Sent Done Done false
127042024-02-15T03:03:02.611Z 7 Acked 1006 Read 4 Sent Done Done false
127052024-02-15T03:03:02.611Z 8 Acked 1007 Write 6 Sent Done Done false
127062024-02-15T03:03:02.611Z 9 Acked 1008 Read 1 Sent Done Done false
127072024-02-15T03:03:02.611Z 10 Acked 1009 Write 3 Sent Done Done false
127082024-02-15T03:03:02.611Z 11 Acked 1010 Read 6 Sent Done Done false
127092024-02-15T03:03:02.612Z 12 Acked 1011 Read 1 Sent Done Done false
127102024-02-15T03:03:02.612Z 13 Acked 1012 Write 2 Sent Done Done false
127112024-02-15T03:03:02.612Z 14 Acked 1013 Read 2 Sent Done Done false
127122024-02-15T03:03:02.612Z 15 Acked 1014 Read 6 Sent Done Done false
127132024-02-15T03:03:02.612Z 16 Acked 1015 Write 2 Sent Done Done false
127142024-02-15T03:03:02.613Z 17 Acked 1016 Write 8 Sent Done Done false
127152024-02-15T03:03:02.613Z 18 Acked 1017 Write 3 Sent Done Done false
127162024-02-15T03:03:02.613Z 19 Acked 1018 Write 9 Sent Done Done false
127172024-02-15T03:03:02.613Z 20 Acked 1019 Read 9 Sent Done Done false
127182024-02-15T03:03:02.613Z 21 Acked 1020 Flush 0 Sent Done Done false
127192024-02-15T03:03:02.613Z 22 Acked 1021 Flush 0 Sent Done Done false
127202024-02-15T03:03:02.614Z 23 Acked 1022 Write 7 Sent Done Done false
127212024-02-15T03:03:02.614Z 24 Acked 1023 Read 6 Sent Done Done false
127222024-02-15T03:03:02.614Z 25 Acked 1024 Write 10 Sent Done Done false
127232024-02-15T03:03:02.614Z 26 Acked 1025 Read 1 Sent Done Done false
127242024-02-15T03:03:02.614Z 27 Acked 1026 Write 1 Sent Done Done false
127252024-02-15T03:03:02.614Z 28 Acked 1027 Write 8 Sent Done Done false
127262024-02-15T03:03:02.615Z 29 Acked 1028 Read 7 Sent Done Done false
127272024-02-15T03:03:02.615Z 30 Acked 1029 Write 8 Sent Sent Sent false
127282024-02-15T03:03:02.615Z STATES DS:0 DS:1 DS:2 TOTAL
127292024-02-15T03:03:02.615Z New 0 0 0 0
127302024-02-15T03:03:02.615Z Sent 28 1 1 30
127312024-02-15T03:03:02.615Z Done 2 29 29 60
127322024-02-15T03:03:02.615Z Skipped 0 0 0 0
127332024-02-15T03:03:02.615Z Error 0 0 0 0
127342024-02-15T03:03:02.616ZLast Flush: 0 1021 1021
127352024-02-15T03:03:02.616ZDownstairs last five completed:
127362024-02-15T03:03:02.616ZUpstairs last five completed: 30 29 28 27 26
127372024-02-15T03:03:02.616Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:01.511462206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1594}
127382024-02-15T03:03:02.616ZWrote out file "/tmp/test_repair_verify.data"
127392024-02-15T03:03:02.622Z
127402024-02-15T03:03:02.622ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127412024-02-15T03:03:02.632ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127422024-02-15T03:03:02.632Z 0 000-019 94 94 94 160 160 160 F F F
127432024-02-15T03:03:02.632Z 1 020-039 110 110 110 186 186 186 F F F
127442024-02-15T03:03:02.632Z 2 040-059 113 113 113 192 192 192 F F F
127452024-02-15T03:03:02.633Z 3 060-079 112 112 112 190 190 190 F F F
127462024-02-15T03:03:02.633Z 4 080-099 113 114 114 192 194 194 F T T <---
127472024-02-15T03:03:02.633Z 5 100-119 113 113 113 192 192 192 F F F
127482024-02-15T03:03:02.633Z 6 120-139 113 113 113 192 192 192 F F F
127492024-02-15T03:03:02.633Z 7 140-159 113 113 113 192 192 192 F F F
127502024-02-15T03:03:02.633Z 8 160-179 110 114 114 186 193 193 T F F <---
127512024-02-15T03:03:02.634Z 9 180-199 110 110 110 187 187 187 F F F
127522024-02-15T03:03:02.634Z 10 200-219 111 111 111 189 189 189 F F F
127532024-02-15T03:03:02.634Z 11 220-239 112 112 112 190 190 190 F F F
127542024-02-15T03:03:02.634Z 12 240-259 111 114 114 189 194 194 F F F <---
127552024-02-15T03:03:02.634Z 13 260-279 108 114 114 184 194 194 F F F <---
127562024-02-15T03:03:02.634Z 14 280-299 108 114 114 184 194 194 F T T <---
127572024-02-15T03:03:02.635Z 15 300-319 99 99 99 167 167 167 F T T <---
127582024-02-15T03:03:02.635Z 16 320-339 108 108 108 184 184 184 F T T <---
127592024-02-15T03:03:02.640Z 17 340-359 105 105 105 179 179 179 F T T <---
127602024-02-15T03:03:02.640Z 18 360-379 113 113 113 192 192 192 F T T <---
127612024-02-15T03:03:02.640Z 19 380-399 106 106 106 181 181 181 F F F
127622024-02-15T03:03:02.640Z 20 400-419 112 114 114 191 194 194 F F F <---
127632024-02-15T03:03:02.641Z 21 420-439 104 104 104 176 176 176 F T T <---
127642024-02-15T03:03:02.641Z 22 440-459 113 113 113 192 192 192 F T T <---
127652024-02-15T03:03:02.641Z 23 460-479 113 114 114 192 193 193 T F F <---
127662024-02-15T03:03:02.641Z 24 480-499 113 113 113 192 192 192 F F F
127672024-02-15T03:03:02.641Z 25 500-519 105 105 105 179 179 179 F F F
127682024-02-15T03:03:02.641Z 26 520-539 106 106 106 181 181 181 F F F
127692024-02-15T03:03:02.641Z 27 540-559 111 111 111 189 189 189 F F F
127702024-02-15T03:03:02.641Z 28 560-579 110 110 110 188 188 188 F F F
127712024-02-15T03:03:02.642Z 29 580-599 113 113 113 192 192 192 F F F
127722024-02-15T03:03:02.642ZMax gen: 114, Max flush: 194
127732024-02-15T03:03:02.642ZError: Difference in extent metadata found!
127742024-02-15T03:03:02.642ZA Difference in extent metadata is expected here
127752024-02-15T03:03:02.642ZOn loop 57
127762024-02-15T03:03:02.642Z
127772024-02-15T03:03:02.642ZVerifying data now
127782024-02-15T03:03:08.364ZNo change after verify
127792024-02-15T03:03:08.364ZLoop: 57 Downstairs dump after verify (and repair):
127802024-02-15T03:03:08.424ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127812024-02-15T03:03:08.424Z 0 000-019 94 94 94 160 160 160 F F F
127822024-02-15T03:03:08.424Z 1 020-039 110 110 110 186 186 186 F F F
127832024-02-15T03:03:08.424Z 2 040-059 113 113 113 192 192 192 F F F
127842024-02-15T03:03:08.424Z 3 060-079 112 112 112 190 190 190 F F F
127852024-02-15T03:03:08.425Z 4 080-099 115 115 115 195 195 195 F F F
127862024-02-15T03:03:08.425Z 5 100-119 113 113 113 192 192 192 F F F
127872024-02-15T03:03:08.425Z 6 120-139 113 113 113 192 192 192 F F F
127882024-02-15T03:03:08.426Z 7 140-159 113 113 113 192 192 192 F F F
127892024-02-15T03:03:08.426Z 8 160-179 114 114 114 193 193 193 F F F
127902024-02-15T03:03:08.426Z 9 180-199 110 110 110 187 187 187 F F F
127912024-02-15T03:03:08.426Z 10 200-219 111 111 111 189 189 189 F F F
127922024-02-15T03:03:08.426Z 11 220-239 112 112 112 190 190 190 F F F
127932024-02-15T03:03:08.426Z 12 240-259 114 114 114 194 194 194 F F F
127942024-02-15T03:03:08.426Z 13 260-279 114 114 114 194 194 194 F F F
127952024-02-15T03:03:08.427Z 14 280-299 115 115 115 195 195 195 F F F
127962024-02-15T03:03:08.428Z 15 300-319 115 115 115 195 195 195 F F F
127972024-02-15T03:03:08.428Z 16 320-339 115 115 115 195 195 195 F F F
127982024-02-15T03:03:08.428Z 17 340-359 115 115 115 195 195 195 F F F
127992024-02-15T03:03:08.428Z 18 360-379 115 115 115 195 195 195 F F F
128002024-02-15T03:03:08.428Z 19 380-399 106 106 106 181 181 181 F F F
128012024-02-15T03:03:08.428Z 20 400-419 114 114 114 194 194 194 F F F
128022024-02-15T03:03:08.428Z 21 420-439 115 115 115 195 195 195 F F F
128032024-02-15T03:03:08.429Z 22 440-459 115 115 115 195 195 195 F F F
128042024-02-15T03:03:08.429Z 23 460-479 114 114 114 193 193 193 F F F
128052024-02-15T03:03:08.429Z 24 480-499 113 113 113 192 192 192 F F F
128062024-02-15T03:03:08.429Z 25 500-519 105 105 105 179 179 179 F F F
128072024-02-15T03:03:08.429Z 26 520-539 106 106 106 181 181 181 F F F
128082024-02-15T03:03:08.429Z 27 540-559 111 111 111 189 189 189 F F F
128092024-02-15T03:03:08.429Z 28 560-579 110 110 110 188 188 188 F F F
128102024-02-15T03:03:08.429Z 29 580-599 113 113 113 192 192 192 F F F
128112024-02-15T03:03:08.429ZMax gen: 115, Max flush: 195
128122024-02-15T03:03:08.430Z
128132024-02-15T03:03:08.430ZBegin loop 58
128142024-02-15T03:03:08.430ZDownstairs to restart: 0
128152024-02-15T03:03:08.430Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.269973022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602}
128162024-02-15T03:03:08.430Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270259376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602}
128172024-02-15T03:03:08.430Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270287652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602}
128182024-02-15T03:03:08.431Z{"msg":"Crucible f7942435-c2c8-4c05-b2c9-3093969996ba has session id: 2e53797d-e98b-4431-950c-48e7a88b3a52","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
128192024-02-15T03:03:08.431Z2024-02-15T03:03:07.270323418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128202024-02-15T03:03:08.431Z{"msg":"Upstairs opts: Upstairs UUID: f7942435-c2c8-4c05-b2c9-3093969996ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270367762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128212024-02-15T03:03:08.431Z{"msg":"Crucible stats registered with UUID: f7942435-c2c8-4c05-b2c9-3093969996ba","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270392919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128222024-02-15T03:03:08.431Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270415476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602}
128232024-02-15T03:03:08.431Z{"msg":"f7942435-c2c8-4c05-b2c9-3093969996ba active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270555769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128242024-02-15T03:03:08.431Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270605343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128252024-02-15T03:03:08.431Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270652457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128262024-02-15T03:03:08.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270681103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128272024-02-15T03:03:08.432Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270721058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128282024-02-15T03:03:08.432Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270746135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128292024-02-15T03:03:08.432Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270780271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128302024-02-15T03:03:08.432Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270817686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128312024-02-15T03:03:08.432Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270840723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128322024-02-15T03:03:08.432Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.270872769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128332024-02-15T03:03:08.432Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:07.270911674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128342024-02-15T03:03:08.433Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:07.270950799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128352024-02-15T03:03:08.433Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:07.270989085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128362024-02-15T03:03:08.437Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271014052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128372024-02-15T03:03:08.437Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271035829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128382024-02-15T03:03:08.437Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271072604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128392024-02-15T03:03:08.437Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271095031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128402024-02-15T03:03:08.437Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271203278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128412024-02-15T03:03:08.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271645942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128422024-02-15T03:03:08.438Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271675269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128432024-02-15T03:03:08.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271690597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128442024-02-15T03:03:08.438Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.271730282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128452024-02-15T03:03:08.438Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.272260545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128462024-02-15T03:03:08.438Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.272413156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128472024-02-15T03:03:08.438Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.272806027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128482024-02-15T03:03:08.438Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.27285697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128492024-02-15T03:03:08.438Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.272916613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128502024-02-15T03:03:08.438Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.272945459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128512024-02-15T03:03:08.439Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.272960288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128522024-02-15T03:03:08.439Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:07.272980855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128532024-02-15T03:03:09.445Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.272524275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128542024-02-15T03:03:09.445Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.272597186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128552024-02-15T03:03:09.477Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.273468197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128562024-02-15T03:03:09.477Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.273512361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128572024-02-15T03:03:09.477Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.273993041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128582024-02-15T03:03:09.477Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274617913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128592024-02-15T03:03:09.478Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274661327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128602024-02-15T03:03:09.478Z{"msg":"[0]R flush_numbers[0..12]: [160, 186, 192, 190, 195, 192, 192, 192, 193, 187, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274687454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128612024-02-15T03:03:09.478Z{"msg":"[0]R generation[0..12]: [94, 110, 113, 112, 115, 113, 113, 113, 114, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274726559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128622024-02-15T03:03:09.478Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274756635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128632024-02-15T03:03:09.478Z{"msg":"[1]R flush_numbers[0..12]: [160, 186, 192, 190, 195, 192, 192, 192, 193, 187, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274785742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128642024-02-15T03:03:09.479Z{"msg":"[1]R generation[0..12]: [94, 110, 113, 112, 115, 113, 113, 113, 114, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.27480052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128652024-02-15T03:03:09.479Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274822597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128662024-02-15T03:03:09.479Z{"msg":"[2]R flush_numbers[0..12]: [160, 186, 192, 190, 195, 192, 192, 192, 193, 187, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274837085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128672024-02-15T03:03:09.480ZWait for a query_work_queue command to finish before sending IO
128682024-02-15T03:03:09.480Z{"msg":"[2]R generation[0..12]: [94, 110, 113, 112, 115, 113, 113, 113, 114, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274865172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128692024-02-15T03:03:09.480Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.27488045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128702024-02-15T03:03:09.481Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274893788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128712024-02-15T03:03:09.481Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274907636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128722024-02-15T03:03:09.481Z{"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274927524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128732024-02-15T03:03:09.481Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274941052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128742024-02-15T03:03:09.481Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274961369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128752024-02-15T03:03:09.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.274974318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128762024-02-15T03:03:09.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.275018842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128772024-02-15T03:03:09.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.27503826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128782024-02-15T03:03:09.482Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.275060007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128792024-02-15T03:03:09.482Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.275073215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128802024-02-15T03:03:09.483Z{"msg":"f7942435-c2c8-4c05-b2c9-3093969996ba is now active with session: 2e53797d-e98b-4431-950c-48e7a88b3a52","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.275087024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128812024-02-15T03:03:09.483Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.275106781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602,"session_id":"2e53797d-e98b-4431-950c-48e7a88b3a52"}
128822024-02-15T03:03:09.483Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:08.27511974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602}
128832024-02-15T03:03:09.484ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128842024-02-15T03:03:09.484ZLoading write count information from file "/tmp/test_repair_verify.data"
128852024-02-15T03:03:09.484ZRun Repair workload
128862024-02-15T03:03:09.484Z01/30 Write block 303 len 36864 data: 14 14 14 11 10 10 12 13 13
128872024-02-15T03:03:09.485Z02/30 Write block 423 len 12288 data: 8 7 7
128882024-02-15T03:03:09.485Z03/30 Flush
128892024-02-15T03:03:09.485Z04/30 Write block 358 len 8192 data: 10 11
128902024-02-15T03:03:09.485Z05/30 Read block 424 len 40960
128912024-02-15T03:03:09.485Z06/30 Write block 188 len 4096 data: 8
128922024-02-15T03:03:09.485Z07/30 Read block 416 len 24576
128932024-02-15T03:03:09.486Z08/30 Read block 330 len 16384
128942024-02-15T03:03:09.486Z09/30 Read block 239 len 40960
128952024-02-15T03:03:09.487Z10/30 Read block 320 len 36864
128962024-02-15T03:03:09.487Z11/30 Write block 418 len 16384 data: 7 8 8 7
128972024-02-15T03:03:09.487Z12/30 Write block 101 len 32768 data: 11 11 9 11 10 8 7 6
128982024-02-15T03:03:09.487Z13/30 Write block 528 len 40960 data: 9 11 11 7 10 8 6 7 6 9
128992024-02-15T03:03:09.487Z14/30 Write block 534 len 8192 data: 7 8
129002024-02-15T03:03:09.488Z15/30 Read block 549 len 28672
129012024-02-15T03:03:09.488Z16/30 Write block 305 len 16384 data: 15 12 11 11
129022024-02-15T03:03:09.488Z17/30 Write block 317 len 24576 data: 9 9 8 7 7 7
129032024-02-15T03:03:09.488Z18/30 Read block 269 len 28672
129042024-02-15T03:03:09.488Z19/30 Read block 215 len 8192
129052024-02-15T03:03:09.489Z20/30 Write block 543 len 36864 data: 12 6 5 7 7 8 7 6 6
129062024-02-15T03:03:09.489Z21/30 Read block 541 len 24576
129072024-02-15T03:03:09.489Z22/30 Write block 67 len 28672 data: 6 6 6 6 6 5 4
129082024-02-15T03:03:09.489Z23/30 Read block 562 len 8192
129092024-02-15T03:03:09.489Z24/30 Read block 131 len 12288
129102024-02-15T03:03:09.489Z25/30 Read block 185 len 32768
129112024-02-15T03:03:09.489Z26/30 Read block 481 len 32768
129122024-02-15T03:03:09.490Z27/30 Read block 181 len 12288
129132024-02-15T03:03:09.490Z28/30 Read block 529 len 40960
129142024-02-15T03:03:09.490Z29/30 Write block 320 len 32768 data: 8 8 8 9 7 9 10 9
129152024-02-15T03:03:09.490Z30/30 Read block 249 len 36864
129162024-02-15T03:03:09.490Z----------------------------------------------------------------
129172024-02-15T03:03:09.490Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:27
129182024-02-15T03:03:09.490ZStates: Active Active Active
129192024-02-15T03:03:09.490ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129202024-02-15T03:03:09.491Z 4 Acked 1003 Write 2 Done Done Done false
129212024-02-15T03:03:09.491Z 5 Acked 1004 Read 10 Done Done Done false
129222024-02-15T03:03:09.491Z 6 Acked 1005 Write 1 Done Done Done false
129232024-02-15T03:03:09.491Z 7 Acked 1006 Read 6 Done Done Done false
129242024-02-15T03:03:09.491Z 8 Acked 1007 Read 4 Done Done Done false
129252024-02-15T03:03:09.491Z 9 Acked 1008 Read 10 Done Done Done false
129262024-02-15T03:03:09.491Z 10 Acked 1009 Read 9 Sent Done Done false
129272024-02-15T03:03:09.491Z 11 Acked 1010 Write 4 Sent Done Done false
129282024-02-15T03:03:09.491Z 12 Acked 1011 Write 8 Sent Done Done false
129292024-02-15T03:03:09.491Z 13 Acked 1012 Write 10 Sent Done Done false
129302024-02-15T03:03:09.491Z 14 Acked 1013 Write 2 Sent Done Done false
129312024-02-15T03:03:09.492Z 15 Acked 1014 Read 7 Sent Done Done false
129322024-02-15T03:03:09.492Z 16 Acked 1015 Write 4 Sent Done Done false
129332024-02-15T03:03:09.492Z 17 Acked 1016 Write 6 Sent Done Done false
129342024-02-15T03:03:09.492Z 18 Acked 1017 Read 7 Sent Done Done false
129352024-02-15T03:03:09.492Z 19 Acked 1018 Read 2 Sent Done Done false
129362024-02-15T03:03:09.492Z 20 Acked 1019 Write 9 Sent Done Done false
129372024-02-15T03:03:09.492Z 21 Acked 1020 Read 6 Sent Done Done false
129382024-02-15T03:03:09.492Z 22 Acked 1021 Write 7 Sent Done Done false
129392024-02-15T03:03:09.492Z 23 Acked 1022 Read 2 Sent Done Done false
129402024-02-15T03:03:09.492Z 24 Acked 1023 Read 3 Sent Done Done false
129412024-02-15T03:03:09.493Z 25 Acked 1024 Read 8 Sent Done Done false
129422024-02-15T03:03:09.493Z 26 Acked 1025 Read 8 Sent Done Done false
129432024-02-15T03:03:09.493Z 27 Acked 1026 Read 3 Sent Done Done false
129442024-02-15T03:03:09.493Z 28 Acked 1027 Read 10 Sent Done Done false
129452024-02-15T03:03:09.493Z 29 Acked 1028 Write 8 Sent Done Done false
129462024-02-15T03:03:09.493Z 30 Acked 1029 Read 9 Sent Done Sent false
129472024-02-15T03:03:09.493Z STATES DS:0 DS:1 DS:2 TOTAL
129482024-02-15T03:03:09.494Z New 0 0 0 0
129492024-02-15T03:03:09.494Z Sent 21 0 1 22
129502024-02-15T03:03:09.494Z Done 6 27 26 59
129512024-02-15T03:03:09.494Z Skipped 0 0 0 0
129522024-02-15T03:03:09.494Z Error 0 0 0 0
129532024-02-15T03:03:09.494ZLast Flush: 1002 1002 1002
129542024-02-15T03:03:09.494ZDownstairs last five completed: 1002 1001 1000
129552024-02-15T03:03:09.517ZUpstairs last five completed: 30 29 28 27 26
129562024-02-15T03:03:09.517Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:08.304862953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1602}
129572024-02-15T03:03:09.517ZWrote out file "/tmp/test_repair_verify.data"
129582024-02-15T03:03:09.517Z
129592024-02-15T03:03:09.517ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
129602024-02-15T03:03:09.518ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129612024-02-15T03:03:09.518Z 0 000-019 94 94 94 160 160 160 F F F
129622024-02-15T03:03:09.518Z 1 020-039 110 110 110 186 186 186 F F F
129632024-02-15T03:03:09.518Z 2 040-059 113 113 113 192 192 192 F F F
129642024-02-15T03:03:09.518Z 3 060-079 112 112 112 190 190 190 F T T <---
129652024-02-15T03:03:09.518Z 4 080-099 115 115 115 195 195 195 F F F
129662024-02-15T03:03:09.518Z 5 100-119 113 113 113 192 192 192 F T T <---
129672024-02-15T03:03:09.518Z 6 120-139 113 113 113 192 192 192 F F F
129682024-02-15T03:03:09.518Z 7 140-159 113 113 113 192 192 192 F F F
129692024-02-15T03:03:09.518Z 8 160-179 114 114 114 193 193 193 F F F
129702024-02-15T03:03:09.518Z 9 180-199 110 110 110 187 187 187 T T T
129712024-02-15T03:03:09.519Z 10 200-219 111 111 111 189 189 189 F F F
129722024-02-15T03:03:09.519Z 11 220-239 112 112 112 190 190 190 F F F
129732024-02-15T03:03:09.519Z 12 240-259 114 114 114 194 194 194 F F F
129742024-02-15T03:03:09.519Z 13 260-279 114 114 114 194 194 194 F F F
129752024-02-15T03:03:09.519Z 14 280-299 115 115 115 195 195 195 F F F
129762024-02-15T03:03:09.519Z 15 300-319 116 116 116 196 196 196 F T T <---
129772024-02-15T03:03:09.519Z 16 320-339 115 115 115 195 195 195 F T T <---
129782024-02-15T03:03:09.519Z 17 340-359 115 115 115 195 195 195 T T T
129792024-02-15T03:03:09.519Z 18 360-379 115 115 115 195 195 195 F F F
129802024-02-15T03:03:09.519Z 19 380-399 106 106 106 181 181 181 F F F
129812024-02-15T03:03:09.520Z 20 400-419 114 114 114 194 194 194 F T T <---
129822024-02-15T03:03:09.520Z 21 420-439 116 116 116 196 196 196 F T T <---
129832024-02-15T03:03:09.520Z 22 440-459 115 115 115 195 195 195 F F F
129842024-02-15T03:03:09.520Z 23 460-479 114 114 114 193 193 193 F F F
129852024-02-15T03:03:09.520Z 24 480-499 113 113 113 192 192 192 F F F
129862024-02-15T03:03:09.520Z 25 500-519 105 105 105 179 179 179 F F F
129872024-02-15T03:03:09.520Z 26 520-539 106 106 106 181 181 181 F T T <---
129882024-02-15T03:03:09.520Z 27 540-559 111 111 111 189 189 189 F T T <---
129892024-02-15T03:03:09.520Z 28 560-579 110 110 110 188 188 188 F F F
129902024-02-15T03:03:09.522Z 29 580-599 113 113 113 192 192 192 F F F
129912024-02-15T03:03:09.522ZMax gen: 116, Max flush: 196
129922024-02-15T03:03:09.522ZError: Difference in extent metadata found!
129932024-02-15T03:03:09.522ZA Difference in extent metadata is expected here
129942024-02-15T03:03:09.523ZOn loop 58
129952024-02-15T03:03:09.523Z
129962024-02-15T03:03:09.523ZVerifying data now
129972024-02-15T03:03:14.991ZNo change after verify
129982024-02-15T03:03:14.991ZLoop: 58 Downstairs dump after verify (and repair):
129992024-02-15T03:03:15.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130002024-02-15T03:03:15.006Z 0 000-019 94 94 94 160 160 160 F F F
130012024-02-15T03:03:15.006Z 1 020-039 110 110 110 186 186 186 F F F
130022024-02-15T03:03:15.006Z 2 040-059 113 113 113 192 192 192 F F F
130032024-02-15T03:03:15.007Z 3 060-079 117 117 117 197 197 197 F F F
130042024-02-15T03:03:15.007Z 4 080-099 115 115 115 195 195 195 F F F
130052024-02-15T03:03:15.007Z 5 100-119 117 117 117 197 197 197 F F F
130062024-02-15T03:03:15.007Z 6 120-139 113 113 113 192 192 192 F F F
130072024-02-15T03:03:15.008Z 7 140-159 113 113 113 192 192 192 F F F
130082024-02-15T03:03:15.008Z 8 160-179 114 114 114 193 193 193 F F F
130092024-02-15T03:03:15.014Z 9 180-199 117 117 117 197 197 197 F F F
130102024-02-15T03:03:15.014Z 10 200-219 111 111 111 189 189 189 F F F
130112024-02-15T03:03:15.014Z 11 220-239 112 112 112 190 190 190 F F F
130122024-02-15T03:03:15.015Z 12 240-259 114 114 114 194 194 194 F F F
130132024-02-15T03:03:15.015Z 13 260-279 114 114 114 194 194 194 F F F
130142024-02-15T03:03:15.015Z 14 280-299 115 115 115 195 195 195 F F F
130152024-02-15T03:03:15.015Z 15 300-319 117 117 117 197 197 197 F F F
130162024-02-15T03:03:15.015Z 16 320-339 117 117 117 197 197 197 F F F
130172024-02-15T03:03:15.015Z 17 340-359 117 117 117 197 197 197 F F F
130182024-02-15T03:03:15.016Z 18 360-379 115 115 115 195 195 195 F F F
130192024-02-15T03:03:15.016Z 19 380-399 106 106 106 181 181 181 F F F
130202024-02-15T03:03:15.016Z 20 400-419 117 117 117 197 197 197 F F F
130212024-02-15T03:03:15.016Z 21 420-439 117 117 117 197 197 197 F F F
130222024-02-15T03:03:15.016Z 22 440-459 115 115 115 195 195 195 F F F
130232024-02-15T03:03:15.016Z 23 460-479 114 114 114 193 193 193 F F F
130242024-02-15T03:03:15.016Z 24 480-499 113 113 113 192 192 192 F F F
130252024-02-15T03:03:15.016Z 25 500-519 105 105 105 179 179 179 F F F
130262024-02-15T03:03:15.016Z 26 520-539 117 117 117 197 197 197 F F F
130272024-02-15T03:03:15.017Z 27 540-559 117 117 117 197 197 197 F F F
130282024-02-15T03:03:15.017Z 28 560-579 110 110 110 188 188 188 F F F
130292024-02-15T03:03:15.017Z 29 580-599 113 113 113 192 192 192 F F F
130302024-02-15T03:03:15.017ZMax gen: 117, Max flush: 197
130312024-02-15T03:03:15.017Z
130322024-02-15T03:03:15.017ZBegin loop 59
130332024-02-15T03:03:15.017ZDownstairs to restart: 1
130342024-02-15T03:03:15.028Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.933631115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610}
130352024-02-15T03:03:15.028Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.933871575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610}
130362024-02-15T03:03:15.046Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.933905501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610}
130372024-02-15T03:03:15.046Z{"msg":"Crucible 30d2437f-6c1d-4bb6-a955-077f9f483b90 has session id: 1ffefe7b-a0a6-4b36-9706-afc8f8cd9263","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.933936507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130382024-02-15T03:03:15.046Z{"msg":"Upstairs opts: Upstairs UUID: 30d2437f-6c1d-4bb6-a955-077f9f483b90, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":Crucible runtime is spawned
130392024-02-15T03:03:15.046Z0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934017077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130402024-02-15T03:03:15.047Z{"msg":"Crucible stats registered with UUID: 30d2437f-6c1d-4bb6-a955-077f9f483b90","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934052063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130412024-02-15T03:03:15.047Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.9340688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610}
130422024-02-15T03:03:15.047Z{"msg":"30d2437f-6c1d-4bb6-a955-077f9f483b90 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934115105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130432024-02-15T03:03:15.047Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934134622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130442024-02-15T03:03:15.047Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934165399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130452024-02-15T03:03:15.047Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934221221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130462024-02-15T03:03:15.047Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934260087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130472024-02-15T03:03:15.047Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934287003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130482024-02-15T03:03:15.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934319539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130492024-02-15T03:03:15.048Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934354015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130502024-02-15T03:03:15.048Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.93438838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130512024-02-15T03:03:15.048Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934426906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130522024-02-15T03:03:15.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934530053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130532024-02-15T03:03:15.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934594435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130542024-02-15T03:03:15.048Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:13.934640239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130552024-02-15T03:03:15.048Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:13.934694612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130562024-02-15T03:03:15.049Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:13.934737307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130572024-02-15T03:03:15.049Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934772462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130582024-02-15T03:03:15.049Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934806258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130592024-02-15T03:03:15.049Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.934924703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130602024-02-15T03:03:15.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.93511165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130612024-02-15T03:03:15.049Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.935143756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130622024-02-15T03:03:15.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.935160174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130632024-02-15T03:03:15.050Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.935181551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130642024-02-15T03:03:15.050Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.935709195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130652024-02-15T03:03:15.050Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.935872274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130662024-02-15T03:03:15.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.936235399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130672024-02-15T03:03:15.050Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.936271644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130682024-02-15T03:03:15.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.936333127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130692024-02-15T03:03:15.051Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.936417046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130702024-02-15T03:03:15.051Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.936451532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130712024-02-15T03:03:15.051Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:13.93646765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130722024-02-15T03:03:16.029Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.936176439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130732024-02-15T03:03:16.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.936301534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130742024-02-15T03:03:16.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.937137209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130752024-02-15T03:03:16.036Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.937175014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130762024-02-15T03:03:16.036Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.93769171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130772024-02-15T03:03:16.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938291094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130782024-02-15T03:03:16.036Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938335009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130792024-02-15T03:03:16.036Z{"msg":"[0]R flush_numbers[0..12]: [160, 186, 192, 197, 195, 197, 192, 192, 193, 197, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938383993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130802024-02-15T03:03:16.036Z{"msg":"[0]R generation[0..12]: [94, 110, 113, 117, 115, 117, 113, 113, 114, 117, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938413839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130812024-02-15T03:03:16.036Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938433217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130822024-02-15T03:03:16.036Z{"msg":"[1]R flush_numbers[0..12]: [160, 186, 192, 197, 195, 197, 192, 192, 193, 197, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938463713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130832024-02-15T03:03:16.036Z{"msg":"[1]R generation[0..12]: [94, 110, 113, 117, 115, 117, 113, 113, 114, 117, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938482341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130842024-02-15T03:03:16.036Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
130852024-02-15T03:03:16.036Z,"time":"2024-02-15T03:03:14.938512657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130862024-02-15T03:03:16.036Z{"msg":"[2]R flush_numbers[0..12]: [160, 186, 192, 197, 195, 197, 192, 192, 193, 197, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938537314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130872024-02-15T03:03:16.036Z{"msg":"[2]R generation[0..12]: [94, 110, 113, 117, 115, 117, 113, 113, 114, 117, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.93856639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130882024-02-15T03:03:16.036Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938584168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130892024-02-15T03:03:16.036Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938613594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130902024-02-15T03:03:16.036Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938630512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130912024-02-15T03:03:16.036Z{"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938653269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130922024-02-15T03:03:16.036Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938676606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130932024-02-15T03:03:16.036Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938707152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130942024-02-15T03:03:16.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.93872927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130952024-02-15T03:03:16.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938760006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130962024-02-15T03:03:16.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938783823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130972024-02-15T03:03:16.037Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.93880862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130982024-02-15T03:03:16.037Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938836946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
130992024-02-15T03:03:16.037Z{"msg":"30d2437f-6c1d-4bb6-a955-077f9f483b90 is now active with session: 1ffefe7b-a0a6-4b36-9706-afc8f8cd9263","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938860363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
131002024-02-15T03:03:16.037Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938898988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610,"session_id":"1ffefe7b-a0a6-4b36-9706-afc8f8cd9263"}
131012024-02-15T03:03:16.037Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:14.938921826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610}
131022024-02-15T03:03:16.037ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131032024-02-15T03:03:16.037ZLoading write count information from file "/tmp/test_repair_verify.data"
131042024-02-15T03:03:16.037ZRun Repair workload
131052024-02-15T03:03:16.037Z01/30 Flush
131062024-02-15T03:03:16.037Z02/30 Write block 509 len 16384 data: 11 10 10 13
131072024-02-15T03:03:16.040Z03/30 Read block 308 len 36864
131082024-02-15T03:03:16.040Z04/30 Write block 369 len 8192 data: 8 8
131092024-02-15T03:03:16.041Z05/30 Read block 70 len 36864
131102024-02-15T03:03:16.041Z06/30 Read block 562 len 8192
131112024-02-15T03:03:16.041Z07/30 Read block 185 len 28672
131122024-02-15T03:03:16.041Z08/30 Write block 385 len 28672 data: 13 12 12 11 12 10 9
131132024-02-15T03:03:16.041Z09/30 Flush
131142024-02-15T03:03:16.044Z10/30 Flush
131152024-02-15T03:03:16.044Z11/30 Write block 61 len 32768 data: 9 10 10 10 7 6 7 7
131162024-02-15T03:03:16.044Z12/30 Read block 569 len 28672
131172024-02-15T03:03:16.047Z13/30 Write block 555 len 36864 data: 10 8 12 13 14 13 9 9 9
131182024-02-15T03:03:16.047Z14/30 Write block 211 len 40960 data: 7 9 9 11 10 9 9 9 11 12
131192024-02-15T03:03:16.047Z15/30 Read block 329 len 40960
131202024-02-15T03:03:16.050Z16/30 Write block 205 len 24576 data: 7 7 7 8 7 6
131212024-02-15T03:03:16.050Z17/30 Read block 431 len 16384
131222024-02-15T03:03:16.052Z18/30 Read block 281 len 4096
131232024-02-15T03:03:16.052Z19/30 Read block 544 len 4096
131242024-02-15T03:03:16.053Z20/30 Read block 224 len 36864
131252024-02-15T03:03:16.053Z21/30 Write block 269 len 32768 data: 7 8 8 10 10 8 8 9
131262024-02-15T03:03:16.055Z22/30 Write block 352 len 4096 data: 11
131272024-02-15T03:03:16.055Z23/30 Read block 394 len 12288
131282024-02-15T03:03:16.056Z24/30 Read block 457 len 8192
131292024-02-15T03:03:16.056Z25/30 Write block 41 len 32768 data: 7 8 7 10 10 9 11 10
131302024-02-15T03:03:16.058Z26/30 Read block 520 len 20480
131312024-02-15T03:03:16.058Z27/30 Write block 395 len 28672 data: 10 8 7 6 8 6 5
131322024-02-15T03:03:16.058Z28/30 Write block 8 len 12288 data: 10 10 12
131332024-02-15T03:03:16.058Z29/30 Read block 258 len 4096
131342024-02-15T03:03:16.062Z30/30 Read block 389 len 8192
131352024-02-15T03:03:16.062Z----------------------------------------------------------------
131362024-02-15T03:03:16.062Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:29
131372024-02-15T03:03:16.062ZStates: Active Active Active
131382024-02-15T03:03:16.062ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131392024-02-15T03:03:16.062Z 2 Acked 1001 Write 4 Done Done Done false
131402024-02-15T03:03:16.063Z 3 Acked 1002 Read 9 Done Done Done false
131412024-02-15T03:03:16.063Z 4 Acked 1003 Write 2 Done Sent Done false
131422024-02-15T03:03:16.063Z 5 Acked 1004 Read 9 Done Sent Done false
131432024-02-15T03:03:16.063Z 6 Acked 1005 Read 2 Done Sent Done false
131442024-02-15T03:03:16.063Z 7 Acked 1006 Read 7 Done Sent Done false
131452024-02-15T03:03:16.063Z 8 Acked 1007 Write 7 Done Sent Done false
131462024-02-15T03:03:16.063Z 9 Acked 1008 Flush 0 Done Sent Done false
131472024-02-15T03:03:16.063Z 10 Acked 1009 Flush 0 Done Sent Done false
131482024-02-15T03:03:16.063Z 11 Acked 1010 Write 8 Done Sent Done false
131492024-02-15T03:03:16.063Z 12 Acked 1011 Read 7 Done Sent Done false
131502024-02-15T03:03:16.063Z 13 Acked 1012 Write 9 Done Sent Done false
131512024-02-15T03:03:16.063Z 14 Acked 1013 Write 10 Done Sent Done false
131522024-02-15T03:03:16.063Z 15 Acked 1014 Read 10 Done Sent Done false
131532024-02-15T03:03:16.063Z 16 Acked 1015 Write 6 Done Sent Done false
131542024-02-15T03:03:16.063Z 17 Acked 1016 Read 4 Done Sent Done false
131552024-02-15T03:03:16.063Z 18 Acked 1017 Read 1 Done Sent Done false
131562024-02-15T03:03:16.063Z 19 Acked 1018 Read 1 Done Sent Done false
131572024-02-15T03:03:16.063Z 20 Acked 1019 Read 9 Done Sent Done false
131582024-02-15T03:03:16.063Z 21 Acked 1020 Write 8 Done Sent Done false
131592024-02-15T03:03:16.063Z 22 Acked 1021 Write 1 Done Sent Done false
131602024-02-15T03:03:16.063Z 23 Acked 1022 Read 3 Done Sent Done false
131612024-02-15T03:03:16.063Z 24 Acked 1023 Read 2 Done Sent Done false
131622024-02-15T03:03:16.063Z 25 Acked 1024 Write 8 Done Sent Done false
131632024-02-15T03:03:16.063Z 26 Acked 1025 Read 5 Done Sent Done false
131642024-02-15T03:03:16.063Z 27 Acked 1026 Write 7 Done Sent Done false
131652024-02-15T03:03:16.064Z 28 Acked 1027 Write 3 Done Sent Done false
131662024-02-15T03:03:16.064Z 29 Acked 1028 Read 1 Done Sent Done false
131672024-02-15T03:03:16.064Z 30 Acked 1029 Read 2 Done Sent Sent false
131682024-02-15T03:03:16.064Z STATES DS:0 DS:1 DS:2 TOTAL
131692024-02-15T03:03:16.064Z New 0 0 0 0
131702024-02-15T03:03:16.064Z Sent 0 27 1 28
131712024-02-15T03:03:16.064Z Done 29 2 28 59
131722024-02-15T03:03:16.064Z Skipped 0 0 0 0
131732024-02-15T03:03:16.064Z Error 0 0 0 0
131742024-02-15T03:03:16.064ZLast Flush: 1009 1000 1009
131752024-02-15T03:03:16.064ZDownstairs last five completed: 1000
131762024-02-15T03:03:16.064ZUpstairs last five completed: 30 29 28 27 26
131772024-02-15T03:03:16.064Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:14.967988324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1610}
131782024-02-15T03:03:16.064ZWrote out file "/tmp/test_repair_verify.data"
131792024-02-15T03:03:16.067Z
131802024-02-15T03:03:16.070ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131812024-02-15T03:03:16.093ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131822024-02-15T03:03:16.093Z 0 000-019 94 94 94 160 160 160 T F T <---
131832024-02-15T03:03:16.093Z 1 020-039 110 110 110 186 186 186 F F F
131842024-02-15T03:03:16.093Z 2 040-059 113 113 113 192 192 192 T F T <---
131852024-02-15T03:03:16.093Z 3 060-079 117 117 117 197 197 197 T F T <---
131862024-02-15T03:03:16.093Z 4 080-099 115 115 115 195 195 195 F F F
131872024-02-15T03:03:16.093Z 5 100-119 117 117 117 197 197 197 F F F
131882024-02-15T03:03:16.093Z 6 120-139 113 113 113 192 192 192 F F F
131892024-02-15T03:03:16.093Z 7 140-159 113 113 113 192 192 192 F F F
131902024-02-15T03:03:16.093Z 8 160-179 114 114 114 193 193 193 F F F
131912024-02-15T03:03:16.093Z 9 180-199 117 117 117 197 197 197 F F F
131922024-02-15T03:03:16.093Z 10 200-219 111 111 111 189 189 189 T F T <---
131932024-02-15T03:03:16.093Z 11 220-239 112 112 112 190 190 190 T F T <---
131942024-02-15T03:03:16.093Z 12 240-259 114 114 114 194 194 194 F F F
131952024-02-15T03:03:16.093Z 13 260-279 114 114 114 194 194 194 T F T <---
131962024-02-15T03:03:16.093Z 14 280-299 115 115 115 195 195 195 F F F
131972024-02-15T03:03:16.093Z 15 300-319 117 117 117 197 197 197 F F F
131982024-02-15T03:03:16.093Z 16 320-339 117 117 117 197 197 197 F F F
131992024-02-15T03:03:16.093Z 17 340-359 117 117 117 197 197 197 T F T <---
132002024-02-15T03:03:16.094Z 18 360-379 118 115 118 199 195 199 F F F <---
132012024-02-15T03:03:16.094Z 19 380-399 118 106 118 199 181 199 T F T <---
132022024-02-15T03:03:16.094Z 20 400-419 117 117 117 197 197 197 T F T <---
132032024-02-15T03:03:16.094Z 21 420-439 117 117 117 197 197 197 F F F
132042024-02-15T03:03:16.094Z 22 440-459 115 115 115 195 195 195 F F F
132052024-02-15T03:03:16.094Z 23 460-479 114 114 114 193 193 193 F F F
132062024-02-15T03:03:16.094Z 24 480-499 113 113 113 192 192 192 F F F
132072024-02-15T03:03:16.094Z 25 500-519 118 105 118 199 179 199 F T F <---
132082024-02-15T03:03:16.094Z 26 520-539 117 117 117 197 197 197 F F F
132092024-02-15T03:03:16.094Z 27 540-559 117 117 117 197 197 197 T F T <---
132102024-02-15T03:03:16.094Z 28 560-579 110 110 110 188 188 188 T F T <---
132112024-02-15T03:03:16.094Z 29 580-599 113 113 113 192 192 192 F F F
132122024-02-15T03:03:16.094ZMax gen: 118, Max flush: 199
132132024-02-15T03:03:16.094ZError: Difference in extent metadata found!
132142024-02-15T03:03:16.103ZA Difference in extent metadata is expected here
132152024-02-15T03:03:16.104ZOn loop 59
132162024-02-15T03:03:16.104Z
132172024-02-15T03:03:16.104ZVerifying data now
132182024-02-15T03:03:21.769ZNo change after verify
132192024-02-15T03:03:21.769ZLoop: 59 Downstairs dump after verify (and repair):
132202024-02-15T03:03:21.785ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132212024-02-15T03:03:21.785Z 0 000-019 119 119 119 200 200 200 F F F
132222024-02-15T03:03:21.785Z 1 020-039 110 110 110 186 186 186 F F F
132232024-02-15T03:03:21.785Z 2 040-059 119 119 119 200 200 200 F F F
132242024-02-15T03:03:21.785Z 3 060-079 119 119 119 200 200 200 F F F
132252024-02-15T03:03:21.785Z 4 080-099 115 115 115 195 195 195 F F F
132262024-02-15T03:03:21.788Z 5 100-119 117 117 117 197 197 197 F F F
132272024-02-15T03:03:21.789Z 6 120-139 113 113 113 192 192 192 F F F
132282024-02-15T03:03:21.789Z 7 140-159 113 113 113 192 192 192 F F F
132292024-02-15T03:03:21.789Z 8 160-179 114 114 114 193 193 193 F F F
132302024-02-15T03:03:21.789Z 9 180-199 117 117 117 197 197 197 F F F
132312024-02-15T03:03:21.789Z 10 200-219 119 119 119 200 200 200 F F F
132322024-02-15T03:03:21.789Z 11 220-239 119 119 119 200 200 200 F F F
132332024-02-15T03:03:21.789Z 12 240-259 114 114 114 194 194 194 F F F
132342024-02-15T03:03:21.789Z 13 260-279 119 119 119 200 200 200 F F F
132352024-02-15T03:03:21.789Z 14 280-299 115 115 115 195 195 195 F F F
132362024-02-15T03:03:21.789Z 15 300-319 117 117 117 197 197 197 F F F
132372024-02-15T03:03:21.789Z 16 320-339 117 117 117 197 197 197 F F F
132382024-02-15T03:03:21.789Z 17 340-359 119 119 119 200 200 200 F F F
132392024-02-15T03:03:21.789Z 18 360-379 118 118 118 199 199 199 F F F
132402024-02-15T03:03:21.789Z 19 380-399 119 119 119 200 200 200 F F F
132412024-02-15T03:03:21.789Z 20 400-419 119 119 119 200 200 200 F F F
132422024-02-15T03:03:21.789Z 21 420-439 117 117 117 197 197 197 F F F
132432024-02-15T03:03:21.789Z 22 440-459 115 115 115 195 195 195 F F F
132442024-02-15T03:03:21.789Z 23 460-479 114 114 114 193 193 193 F F F
132452024-02-15T03:03:21.789Z 24 480-499 113 113 113 192 192 192 F F F
132462024-02-15T03:03:21.789Z 25 500-519 118 118 118 199 199 199 F F F
132472024-02-15T03:03:21.789Z 26 520-539 117 117 117 197 197 197 F F F
132482024-02-15T03:03:21.789Z 27 540-559 119 119 119 200 200 200 F F F
132492024-02-15T03:03:21.789Z 28 560-579 119 119 119 200 200 200 F F F
132502024-02-15T03:03:21.790Z 29 580-599 113 113 113 192 192 192 F F F
132512024-02-15T03:03:21.790ZMax gen: 119, Max flush: 200
132522024-02-15T03:03:21.799Z
132532024-02-15T03:03:21.799ZBegin loop 60
132542024-02-15T03:03:21.799ZDownstairs to restart: 1
132552024-02-15T03:03:21.823Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.714951237Z"Crucible runtime is spawned
132562024-02-15T03:03:21.823Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618}
132572024-02-15T03:03:21.823Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715346347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618}
132582024-02-15T03:03:21.823Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715387432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618}
132592024-02-15T03:03:21.823Z{"msg":"Crucible d970bac4-2684-4f83-9ea8-98904dd35b53 has session id: e74b616b-c1c1-4759-af3f-1c9dc898c249","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715411179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132602024-02-15T03:03:21.823Z{"msg":"Upstairs opts: Upstairs UUID: d970bac4-2684-4f83-9ea8-98904dd35b53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715463473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132612024-02-15T03:03:21.823Z{"msg":"Crucible stats registered with UUID: d970bac4-2684-4f83-9ea8-98904dd35b53","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715506017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132622024-02-15T03:03:21.823Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715546152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618}
132632024-02-15T03:03:21.823Z{"msg":"d970bac4-2684-4f83-9ea8-98904dd35b53 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715583948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132642024-02-15T03:03:21.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715622733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132652024-02-15T03:03:21.824Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715656179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132662024-02-15T03:03:21.824Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715684115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132672024-02-15T03:03:21.824Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715708452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132682024-02-15T03:03:21.824Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715730069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132692024-02-15T03:03:21.824Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715752496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132702024-02-15T03:03:21.824Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715781373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132712024-02-15T03:03:21.824Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715825247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132722024-02-15T03:03:21.824Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715854164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132732024-02-15T03:03:21.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715890769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132742024-02-15T03:03:21.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.715916306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132752024-02-15T03:03:21.824Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:20.716085985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132762024-02-15T03:03:21.824Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:20.716139898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132772024-02-15T03:03:21.824Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:20.716210959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132782024-02-15T03:03:21.824Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.716247325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132792024-02-15T03:03:21.824Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.716275031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132802024-02-15T03:03:21.824Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.716300518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132812024-02-15T03:03:21.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.716501723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132822024-02-15T03:03:21.824Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.716540928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132832024-02-15T03:03:21.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.716592621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132842024-02-15T03:03:21.824Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.716624257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132852024-02-15T03:03:21.824Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717141733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132862024-02-15T03:03:21.824Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717285734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132872024-02-15T03:03:21.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717656038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132882024-02-15T03:03:21.825Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717706522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132892024-02-15T03:03:21.825Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717785042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132902024-02-15T03:03:21.825Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717825747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132912024-02-15T03:03:21.825Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717843265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132922024-02-15T03:03:21.825Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:20.717864702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132932024-02-15T03:03:22.812Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.717731442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132942024-02-15T03:03:22.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.717862685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132952024-02-15T03:03:22.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.7187051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132962024-02-15T03:03:22.821Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.718751344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132972024-02-15T03:03:22.821Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.719255881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132982024-02-15T03:03:22.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.719861215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
132992024-02-15T03:03:22.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.719946404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133002024-02-15T03:03:22.821Z{"msg":"[0]R flush_numbers[0..12]: [200, 186, 200, 200, 195, 197, 192, 192, 193, 197, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.71998188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133012024-02-15T03:03:22.821Z{"msg":"[0]R generation[0..12]: [119, 110, 119, 119, 115, 117, 113, 113, 114, 117, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720022585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133022024-02-15T03:03:22.821Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720044892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133032024-02-15T03:03:22.822Z{"msg":"[1]R flush_numbers[0..12]: [200, 186, 200, 200, 195, 197, 192, 192, 193, 197, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720072169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133042024-02-15T03:03:22.822Z{"msg":"[1]R generation[0..12]: [119, 110, 119, 119, 115, 117, 113, 113, 114, 117, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720090636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":Wait for a query_work_queue command to finish before sending IO
133052024-02-15T03:03:22.822Z"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133062024-02-15T03:03:22.822Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720124532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133072024-02-15T03:03:22.822Z{"msg":"[2]R flush_numbers[0..12]: [200, 186, 200, 200, 195, 197, 192, 192, 193, 197, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.72014322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133082024-02-15T03:03:22.822Z{"msg":"[2]R generation[0..12]: [119, 110, 119, 119, 115, 117, 113, 113, 114, 117, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720169186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133092024-02-15T03:03:22.822Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720188774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133102024-02-15T03:03:22.822Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720205562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133112024-02-15T03:03:22.823Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720231518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133122024-02-15T03:03:22.823Z{"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720248287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133132024-02-15T03:03:22.823Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720265304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133142024-02-15T03:03:22.823Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720282982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133152024-02-15T03:03:22.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.72029825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133162024-02-15T03:03:22.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720325807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133172024-02-15T03:03:22.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720342955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133182024-02-15T03:03:22.824Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720368112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133192024-02-15T03:03:22.824Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720385489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133202024-02-15T03:03:22.824Z{"msg":"d970bac4-2684-4f83-9ea8-98904dd35b53 is now active with session: e74b616b-c1c1-4759-af3f-1c9dc898c249","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720403237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133212024-02-15T03:03:22.824Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720418885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618,"session_id":"e74b616b-c1c1-4759-af3f-1c9dc898c249"}
133222024-02-15T03:03:22.824Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:21.720434263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618}
133232024-02-15T03:03:22.824ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133242024-02-15T03:03:22.824ZLoading write count information from file "/tmp/test_repair_verify.data"
133252024-02-15T03:03:22.824ZRun Repair workload
133262024-02-15T03:03:22.825Z01/30 Write block 241 len 4096 data: 11
133272024-02-15T03:03:22.830Z02/30 Write block 25 len 24576 data: 4 6 7 8 7 8
133282024-02-15T03:03:22.830Z03/30 Read block 472 len 24576
133292024-02-15T03:03:22.830Z04/30 Write block 470 len 4096 data: 12
133302024-02-15T03:03:22.830Z05/30 Read block 396 len 4096
133312024-02-15T03:03:22.830Z06/30 Read block 473 len 4096
133322024-02-15T03:03:22.831Z07/30 Read block 21 len 20480
133332024-02-15T03:03:22.831Z08/30 Write block 516 len 40960 data: 7 6 5 4 4 3 2 2 2 3
133342024-02-15T03:03:22.831Z09/30 Write block 58 len 36864 data: 12 9 9 10 11 11 11 8 7
133352024-02-15T03:03:22.831Z10/30 Flush
133362024-02-15T03:03:22.831Z11/30 Write block 516 len 32768 data: 8 7 6 5 5 4 3 3
133372024-02-15T03:03:22.831Z12/30 Read block 201 len 20480
133382024-02-15T03:03:22.831Z13/30 Read block 389 len 28672
133392024-02-15T03:03:22.838Z14/30 Write block 432 len 40960 data: 7 7 7 9 11 12 12 10 11 11
133402024-02-15T03:03:22.838Z15/30 Write block 425 len 36864 data: 8 9 9 8 7 7 7 8 8
133412024-02-15T03:03:22.838Z16/30 Read block 587 len 12288
133422024-02-15T03:03:22.838Z17/30 Flush
133432024-02-15T03:03:22.838Z18/30 Flush
133442024-02-15T03:03:22.843Z19/30 Write block 377 len 12288 data: 10 11 8
133452024-02-15T03:03:22.843Z20/30 Read block 402 len 36864
133462024-02-15T03:03:22.843Z21/30 Write block 295 len 24576 data: 11 10 9 9 10 9
133472024-02-15T03:03:22.843Z22/30 Write block 35 len 20480 data: 9 8 8 9 7
133482024-02-15T03:03:22.844Z23/30 Write block 141 len 8192 data: 13 11
133492024-02-15T03:03:22.844Z24/30 Write block 256 len 20480 data: 8 10 10 11 8
133502024-02-15T03:03:22.844Z25/30 Read block 105 len 16384
133512024-02-15T03:03:22.844Z26/30 Read block 242 len 24576
133522024-02-15T03:03:22.844Z27/30 Write block 388 len 28672 data: 12 13 11 10 9 8 9
133532024-02-15T03:03:22.844Z28/30 Read block 420 len 12288
133542024-02-15T03:03:22.882Z29/30 Read block 2 len 16384
133552024-02-15T03:03:22.883Z30/30 Write block 349 len 16384 data: 9 10 11 12
133562024-02-15T03:03:22.883Z----------------------------------------------------------------
133572024-02-15T03:03:22.883Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
133582024-02-15T03:03:22.883ZStates: Active Active Active
133592024-02-15T03:03:22.883ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133602024-02-15T03:03:22.883Z 1 Acked 1000 Write 1 Done Sent Done false
133612024-02-15T03:03:22.883Z 2 Acked 1001 Write 6 Done Sent Done false
133622024-02-15T03:03:22.883Z 3 Acked 1002 Read 6 Done Sent Done false
133632024-02-15T03:03:22.883Z 4 Acked 1003 Write 1 Done Sent Done false
133642024-02-15T03:03:22.883Z 5 Acked 1004 Read 1 Done Sent Done false
133652024-02-15T03:03:22.884Z 6 Acked 1005 Read 1 Done Sent Done false
133662024-02-15T03:03:22.884Z 7 Acked 1006 Read 5 Done Sent Done false
133672024-02-15T03:03:22.884Z 8 Acked 1007 Write 10 Done Sent Done false
133682024-02-15T03:03:22.884Z 9 Acked 1008 Write 9 Done Sent Done false
133692024-02-15T03:03:22.884Z 10 Acked 1009 Flush 0 Done Sent Done false
133702024-02-15T03:03:22.884Z 11 Acked 1010 Write 8 Done Sent Done false
133712024-02-15T03:03:22.884Z 12 Acked 1011 Read 5 Done Sent Done false
133722024-02-15T03:03:22.884Z 13 Acked 1012 Read 7 Done Sent Done false
133732024-02-15T03:03:22.884Z 14 Acked 1013 Write 10 Done Sent Done false
133742024-02-15T03:03:22.884Z 15 Acked 1014 Write 9 Done Sent Done false
133752024-02-15T03:03:22.884Z 16 Acked 1015 Read 3 Done Sent Done false
133762024-02-15T03:03:22.884Z 17 Acked 1016 Flush 0 Done Sent Done false
133772024-02-15T03:03:22.884Z 18 Acked 1017 Flush 0 Done Sent Done false
133782024-02-15T03:03:22.884Z 19 Acked 1018 Write 3 Done Sent Done false
133792024-02-15T03:03:22.884Z 20 Acked 1019 Read 9 Done Sent Done false
133802024-02-15T03:03:22.885Z 21 Acked 1020 Write 6 Done Sent Done false
133812024-02-15T03:03:22.885Z 22 Acked 1021 Write 5 Done Sent Done false
133822024-02-15T03:03:22.885Z 23 Acked 1022 Write 2 Done Sent Done false
133832024-02-15T03:03:22.885Z 24 Acked 1023 Write 5 Done Sent Done false
133842024-02-15T03:03:22.885Z 25 Acked 1024 Read 4 Done Sent Done false
133852024-02-15T03:03:22.885Z 26 Acked 1025 Read 6 Done Sent Done false
133862024-02-15T03:03:22.885Z 27 Acked 1026 Write 7 Done Sent Done false
133872024-02-15T03:03:22.885Z 28 Acked 1027 Read 3 Done Sent Done false
133882024-02-15T03:03:22.885Z 29 Acked 1028 Read 4 Done Sent Done false
133892024-02-15T03:03:22.885Z 30 Acked 1029 Write 4 Sent Sent Sent false
133902024-02-15T03:03:22.885Z STATES DS:0 DS:1 DS:2 TOTAL
133912024-02-15T03:03:22.885Z New 0 0 0 0
133922024-02-15T03:03:22.885Z Sent 1 30 1 32
133932024-02-15T03:03:22.885Z Done 29 0 29 58
133942024-02-15T03:03:22.886Z Skipped 0 0 0 0
133952024-02-15T03:03:22.886Z Error 0 0 0 0
133962024-02-15T03:03:22.886ZLast Flush: 1017 0 1017
133972024-02-15T03:03:22.886ZDownstairs last five completed:
133982024-02-15T03:03:22.886ZUpstairs last five completed: 30 29 28 27 26
133992024-02-15T03:03:22.886Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:21.751760338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1618}
134002024-02-15T03:03:22.886ZWrote out file "/tmp/test_repair_verify.data"
134012024-02-15T03:03:22.886Z
134022024-02-15T03:03:22.887ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134032024-02-15T03:03:22.887ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134042024-02-15T03:03:22.887Z 0 000-019 119 119 119 200 200 200 F F F
134052024-02-15T03:03:22.887Z 1 020-039 120 110 120 201 186 201 T F T <---
134062024-02-15T03:03:22.887Z 2 040-059 120 119 120 201 200 201 F F F <---
134072024-02-15T03:03:22.887Z 3 060-079 120 119 120 201 200 201 F F F <---
134082024-02-15T03:03:22.887Z 4 080-099 115 115 115 195 195 195 F F F
134092024-02-15T03:03:22.887Z 5 100-119 117 117 117 197 197 197 F F F
134102024-02-15T03:03:22.887Z 6 120-139 113 113 113 192 192 192 F F F
134112024-02-15T03:03:22.887Z 7 140-159 113 113 113 192 192 192 T F T <---
134122024-02-15T03:03:22.888Z 8 160-179 114 114 114 193 193 193 F F F
134132024-02-15T03:03:22.888Z 9 180-199 117 117 117 197 197 197 F F F
134142024-02-15T03:03:22.888Z 10 200-219 119 119 119 200 200 200 F F F
134152024-02-15T03:03:22.888Z 11 220-239 119 119 119 200 200 200 F F F
134162024-02-15T03:03:22.888Z 12 240-259 120 114 120 201 194 201 T F T <---
134172024-02-15T03:03:22.888Z 13 260-279 119 119 119 200 200 200 T F T <---
134182024-02-15T03:03:22.888Z 14 280-299 115 115 115 195 195 195 T F T <---
134192024-02-15T03:03:22.888Z 15 300-319 117 117 117 197 197 197 T F T <---
134202024-02-15T03:03:22.888Z 16 320-339 117 117 117 197 197 197 F F F
134212024-02-15T03:03:22.889Z 17 340-359 119 119 119 200 200 200 T F T <---
134222024-02-15T03:03:22.889Z 18 360-379 118 118 118 199 199 199 T F T <---
134232024-02-15T03:03:22.889Z 19 380-399 119 119 119 200 200 200 T F T <---
134242024-02-15T03:03:22.889Z 20 400-419 119 119 119 200 200 200 F F F
134252024-02-15T03:03:22.889Z 21 420-439 120 117 120 202 197 202 F F F <---
134262024-02-15T03:03:22.889Z 22 440-459 120 115 120 202 195 202 F F F <---
134272024-02-15T03:03:22.889Z 23 460-479 120 114 120 201 193 201 F F F <---
134282024-02-15T03:03:22.889Z 24 480-499 113 113 113 192 192 192 F F F
134292024-02-15T03:03:22.889Z 25 500-519 120 118 120 202 199 202 F F F <---
134302024-02-15T03:03:22.889Z 26 520-539 120 117 120 202 197 202 F F F <---
134312024-02-15T03:03:22.889Z 27 540-559 119 119 119 200 200 200 F F F
134322024-02-15T03:03:22.889Z 28 560-579 119 119 119 200 200 200 F F F
134332024-02-15T03:03:22.890Z 29 580-599 113 113 113 192 192 192 F F F
134342024-02-15T03:03:22.890ZMax gen: 120, Max flush: 202
134352024-02-15T03:03:22.890ZError: Difference in extent metadata found!
134362024-02-15T03:03:22.890ZA Difference in extent metadata is expected here
134372024-02-15T03:03:22.890ZOn loop 60
134382024-02-15T03:03:22.890Z
134392024-02-15T03:03:22.894ZVerifying data now
134402024-02-15T03:03:27.681ZNo change after verify
134412024-02-15T03:03:27.681ZLoop: 60 Downstairs dump after verify (and repair):
134422024-02-15T03:03:27.695ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134432024-02-15T03:03:27.695Z 0 000-019 119 119 119 200 200 200 F F F
134442024-02-15T03:03:27.695Z 1 020-039 121 121 121 203 203 203 F F F
134452024-02-15T03:03:27.695Z 2 040-059 120 120 120 201 201 201 F F F
134462024-02-15T03:03:27.695Z 3 060-079 120 120 120 201 201 201 F F F
134472024-02-15T03:03:27.696Z 4 080-099 115 115 115 195 195 195 F F F
134482024-02-15T03:03:27.696Z 5 100-119 117 117 117 197 197 197 F F F
134492024-02-15T03:03:27.696Z 6 120-139 113 113 113 192 192 192 F F F
134502024-02-15T03:03:27.696Z 7 140-159 121 121 121 203 203 203 F F F
134512024-02-15T03:03:27.696Z 8 160-179 114 114 114 193 193 193 F F F
134522024-02-15T03:03:27.696Z 9 180-199 117 117 117 197 197 197 F F F
134532024-02-15T03:03:27.696Z 10 200-219 119 119 119 200 200 200 F F F
134542024-02-15T03:03:27.696Z 11 220-239 119 119 119 200 200 200 F F F
134552024-02-15T03:03:27.697Z 12 240-259 121 121 121 203 203 203 F F F
134562024-02-15T03:03:27.697Z 13 260-279 121 121 121 203 203 203 F F F
134572024-02-15T03:03:27.697Z 14 280-299 121 121 121 203 203 203 F F F
134582024-02-15T03:03:27.697Z 15 300-319 121 121 121 203 203 203 F F F
134592024-02-15T03:03:27.697Z 16 320-339 117 117 117 197 197 197 F F F
134602024-02-15T03:03:27.697Z 17 340-359 121 121 121 203 203 203 F F F
134612024-02-15T03:03:27.697Z 18 360-379 121 121 121 203 203 203 F F F
134622024-02-15T03:03:27.697Z 19 380-399 121 121 121 203 203 203 F F F
134632024-02-15T03:03:27.697Z 20 400-419 119 119 119 200 200 200 F F F
134642024-02-15T03:03:27.697Z 21 420-439 120 120 120 202 202 202 F F F
134652024-02-15T03:03:27.698Z 22 440-459 120 120 120 202 202 202 F F F
134662024-02-15T03:03:27.698Z 23 460-479 120 120 120 201 201 201 F F F
134672024-02-15T03:03:27.698Z 24 480-499 113 113 113 192 192 192 F F F
134682024-02-15T03:03:27.698Z 25 500-519 120 120 120 202 202 202 F F F
134692024-02-15T03:03:27.698Z 26 520-539 120 120 120 202 202 202 F F F
134702024-02-15T03:03:27.698Z 27 540-559 119 119 119 200 200 200 F F F
134712024-02-15T03:03:27.698Z 28 560-579 119 119 119 200 200 200 F F F
134722024-02-15T03:03:27.698Z 29 580-599 113 113 113 192 192 192 F F F
134732024-02-15T03:03:27.698ZMax gen: 121, Max flush: 203
134742024-02-15T03:03:27.709Z
134752024-02-15T03:03:27.709ZBegin loop 61
134762024-02-15T03:03:27.709ZDownstairs to restart: 2
134772024-02-15T03:03:27.728Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619271153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626}
134782024-02-15T03:03:27.728Z{Crucible runtime is spawned
134792024-02-15T03:03:27.728Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619688151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626}
134802024-02-15T03:03:27.728Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619745844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626}
134812024-02-15T03:03:27.728Z{"msg":"Crucible dbbe0bfb-1195-4cbf-82ea-0147f4d5aa3a has session id: 42dc6bfd-fb7d-462d-83e0-45ea84267437","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619782039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134822024-02-15T03:03:27.728Z{"msg":"Upstairs opts: Upstairs UUID: dbbe0bfb-1195-4cbf-82ea-0147f4d5aa3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619828184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134832024-02-15T03:03:27.728Z{"msg":"Crucible stats registered with UUID: dbbe0bfb-1195-4cbf-82ea-0147f4d5aa3a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.61985643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134842024-02-15T03:03:27.728Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619906444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626}
134852024-02-15T03:03:27.728Z{"msg":"dbbe0bfb-1195-4cbf-82ea-0147f4d5aa3a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619951178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134862024-02-15T03:03:27.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.619979794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134872024-02-15T03:03:27.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620010421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134882024-02-15T03:03:27.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620049906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134892024-02-15T03:03:27.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620078652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134902024-02-15T03:03:27.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620112948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134912024-02-15T03:03:27.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620180149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134922024-02-15T03:03:27.729Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620250911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134932024-02-15T03:03:27.730Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620321612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134942024-02-15T03:03:27.730Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620361437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134952024-02-15T03:03:27.730Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.62041733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134962024-02-15T03:03:27.730Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620441687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134972024-02-15T03:03:27.730Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:26.620473723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134982024-02-15T03:03:27.730Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:26.62049172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
134992024-02-15T03:03:27.730Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:26.620526576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135002024-02-15T03:03:27.730Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620546803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135012024-02-15T03:03:27.730Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.62057568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135022024-02-15T03:03:27.730Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.620774935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135032024-02-15T03:03:27.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.621139789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135042024-02-15T03:03:27.731Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.621176175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135052024-02-15T03:03:27.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.621194312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135062024-02-15T03:03:27.731Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.621225518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135072024-02-15T03:03:27.742Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.621796297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135082024-02-15T03:03:27.742Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.621834502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135092024-02-15T03:03:27.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.622391372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135102024-02-15T03:03:27.742Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.622435897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135112024-02-15T03:03:27.742Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.622468573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135122024-02-15T03:03:27.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.622483941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135132024-02-15T03:03:27.742Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.622508638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135142024-02-15T03:03:27.742Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:26.622543803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135152024-02-15T03:03:28.719Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.622381597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135162024-02-15T03:03:28.720Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.622473105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135172024-02-15T03:03:28.737Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.623406219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135182024-02-15T03:03:28.737Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.623457112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135192024-02-15T03:03:28.737Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.623959229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135202024-02-15T03:03:28.738Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624610377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135212024-02-15T03:03:28.738Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624643963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135222024-02-15T03:03:28.738Z{"msg":"[0]R flush_numbers[0..12]: [200, 203, 201, 201, 195, 197, 192, 203, 193, 197, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624663101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135232024-02-15T03:03:28.738Z{"msg":"[0]R generation[0..12]: [119, 121, 120, 120, 115, 117, 113, 121, 114, 117, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624681249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135242024-02-15T03:03:28.738Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624781496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135252024-02-15T03:03:28.738Z{"msg":"[1]R flush_numbers[0..12]: [200, 203, 201, 201, 195, 197, 192, 203, 193, 197, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624815792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135262024-02-15T03:03:28.738Z{"msg":"[1]R generation[0..12]: [119, 121, 120, 120, 115, 117, 113, 121, 114, 117, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.62483458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135272024-02-15T03:03:28.739Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624861316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135282024-02-15T03:03:28.739Z{"msg":"[2]R flush_numbers[0..12]: [200, 203, 201, 201, 195, 197, 192, 203, 193, 197, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624880274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135292024-02-15T03:03:28.739Z{"msg":"[2]R generation[0..12]: [119, 121, 120, 120, 115, 117, 113, 121, 114, 117, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624905771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135302024-02-15T03:03:28.739Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
135312024-02-15T03:03:28.739Z,"time":"2024-02-15T03:03:27.624923798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135322024-02-15T03:03:28.740Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624951785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135332024-02-15T03:03:28.740Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.624969852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135342024-02-15T03:03:28.740Z{"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.62499251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135352024-02-15T03:03:28.740Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625006208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135362024-02-15T03:03:28.741Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625019556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135372024-02-15T03:03:28.741Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625032835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135382024-02-15T03:03:28.741Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625052892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135392024-02-15T03:03:28.741Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.62506654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135402024-02-15T03:03:28.741Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625089897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135412024-02-15T03:03:28.741Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625106615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135422024-02-15T03:03:28.741Z{"msg":"dbbe0bfb-1195-4cbf-82ea-0147f4d5aa3a is now active with session: 42dc6bfd-fb7d-462d-83e0-45ea84267437","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625121173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135432024-02-15T03:03:28.742Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.62514601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626,"session_id":"42dc6bfd-fb7d-462d-83e0-45ea84267437"}
135442024-02-15T03:03:28.742Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:27.625159549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626}
135452024-02-15T03:03:28.742ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135462024-02-15T03:03:28.742ZLoading write count information from file "/tmp/test_repair_verify.data"
135472024-02-15T03:03:28.743ZRun Repair workload
135482024-02-15T03:03:28.743Z01/30 Flush
135492024-02-15T03:03:28.743Z02/30 Write block 137 len 8192 data: 10 11
135502024-02-15T03:03:28.743Z03/30 Write block 413 len 4096 data: 18
135512024-02-15T03:03:28.744Z04/30 Read block 537 len 8192
135522024-02-15T03:03:28.744Z05/30 Read block 306 len 12288
135532024-02-15T03:03:28.744Z06/30 Write block 589 len 4096 data: 11
135542024-02-15T03:03:28.744Z07/30 Read block 424 len 20480
135552024-02-15T03:03:28.745Z08/30 Read block 122 len 28672
135562024-02-15T03:03:28.745Z09/30 Read block 304 len 4096
135572024-02-15T03:03:28.762Z10/30 Write block 89 len 36864 data: 7 8 8 7 6 10 9 12 14
135582024-02-15T03:03:28.762Z11/30 Write block 499 len 32768 data: 9 9 9 11 12 11 10 10
135592024-02-15T03:03:28.762Z12/30 Read block 37 len 32768
135602024-02-15T03:03:28.762Z13/30 Read block 462 len 24576
135612024-02-15T03:03:28.762Z14/30 Write block 20 len 4096 data: 5
135622024-02-15T03:03:28.762Z15/30 Read block 579 len 8192
135632024-02-15T03:03:28.762Z16/30 Read block 247 len 36864
135642024-02-15T03:03:28.762Z17/30 Read block 382 len 40960
135652024-02-15T03:03:28.762Z18/30 Read block 72 len 16384
135662024-02-15T03:03:28.763Z19/30 Write block 215 len 16384 data: 11 10 10 10
135672024-02-15T03:03:28.763Z20/30 Write block 527 len 12288 data: 8 10 12
135682024-02-15T03:03:28.763Z21/30 Write block 420 len 24576 data: 9 8 7 9 8 9
135692024-02-15T03:03:28.763Z22/30 Write block 492 len 40960 data: 12 10 11 12 12 12 12 10 10 10
135702024-02-15T03:03:28.763Z23/30 Write block 267 len 24576 data: 6 7 8 9 9 11
135712024-02-15T03:03:28.763Z24/30 Read block 0 len 4096
135722024-02-15T03:03:28.763Z25/30 Write block 565 len 36864 data: 10 5 4 4 4 7 9 9 13
135732024-02-15T03:03:28.763Z26/30 Write block 17 len 24576 data: 6 8 7 6 5 4
135742024-02-15T03:03:28.763Z27/30 Write block 329 len 12288 data: 10 11 12
135752024-02-15T03:03:28.763Z28/30 Write block 523 len 8192 data: 4 3
135762024-02-15T03:03:28.763Z29/30 Write block 396 len 28672 data: 9 8 7 9 7 6 5
135772024-02-15T03:03:28.763Z30/30 Write block 289 len 24576 data: 10 17 15 14 13 13
135782024-02-15T03:03:28.763Z----------------------------------------------------------------
135792024-02-15T03:03:28.763Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:29
135802024-02-15T03:03:28.763ZStates: Active Active Active
135812024-02-15T03:03:28.764ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135822024-02-15T03:03:28.764Z 2 Acked 1001 Write 2 Done Done Done false
135832024-02-15T03:03:28.764Z 3 Acked 1002 Write 1 Done Done Done false
135842024-02-15T03:03:28.764Z 4 Acked 1003 Read 2 Done Done Done false
135852024-02-15T03:03:28.764Z 5 Acked 1004 Read 3 Done Done Sent false
135862024-02-15T03:03:28.764Z 6 Acked 1005 Write 1 Done Done Sent false
135872024-02-15T03:03:28.764Z 7 Acked 1006 Read 5 Done Done Sent false
135882024-02-15T03:03:28.764Z 8 Acked 1007 Read 7 Done Done Sent false
135892024-02-15T03:03:28.764Z 9 Acked 1008 Read 1 Done Done Sent false
135902024-02-15T03:03:28.764Z 10 Acked 1009 Write 9 Done Done Sent false
135912024-02-15T03:03:28.764Z 11 Acked 1010 Write 8 Done Done Sent false
135922024-02-15T03:03:28.764Z 12 Acked 1011 Read 8 Done Done Sent false
135932024-02-15T03:03:28.764Z 13 Acked 1012 Read 6 Done Done Sent false
135942024-02-15T03:03:28.764Z 14 Acked 1013 Write 1 Done Done Sent false
135952024-02-15T03:03:28.764Z 15 Acked 1014 Read 2 Done Done Sent false
135962024-02-15T03:03:28.765Z 16 Acked 1015 Read 9 Done Done Sent false
135972024-02-15T03:03:28.765Z 17 Acked 1016 Read 10 Done Done Sent false
135982024-02-15T03:03:28.765Z 18 Acked 1017 Read 4 Done Done Sent false
135992024-02-15T03:03:28.765Z 19 Acked 1018 Write 4 Done Done Sent false
136002024-02-15T03:03:28.765Z 20 Acked 1019 Write 3 Done Done Sent false
136012024-02-15T03:03:28.765Z 21 Acked 1020 Write 6 Done Done Sent false
136022024-02-15T03:03:28.765Z 22 Acked 1021 Write 10 Done Done Sent false
136032024-02-15T03:03:28.765Z 23 Acked 1022 Write 6 Done Done Sent false
136042024-02-15T03:03:28.765Z 24 Acked 1023 Read 1 Done Done Sent false
136052024-02-15T03:03:28.765Z 25 Acked 1024 Write 9 Done Done Sent false
136062024-02-15T03:03:28.765Z 26 Acked 1025 Write 6 Done Done Sent false
136072024-02-15T03:03:28.765Z 27 Acked 1026 Write 3 Done Done Sent false
136082024-02-15T03:03:28.766Z 28 Acked 1027 Write 2 Done Done Sent false
136092024-02-15T03:03:28.766Z 29 Acked 1028 Write 7 Sent Sent Sent false
136102024-02-15T03:03:28.766Z 30 Acked 1029 Write 6 Sent Sent Sent false
136112024-02-15T03:03:28.766Z STATES DS:0 DS:1 DS:2 TOTAL
136122024-02-15T03:03:28.766Z New 0 0 0 0
136132024-02-15T03:03:28.766Z Sent 2 2 26 30
136142024-02-15T03:03:28.766Z Done 27 27 3 57
136152024-02-15T03:03:28.766Z Skipped 0 0 0 0
136162024-02-15T03:03:28.766Z Error 0 0 0 0
136172024-02-15T03:03:28.766ZLast Flush: 1000 1000 1000
136182024-02-15T03:03:28.766ZDownstairs last five completed: 1000
136192024-02-15T03:03:28.766ZUpstairs last five completed: 30 29 28 27 26
136202024-02-15T03:03:28.766Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:27.651019469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1626}
136212024-02-15T03:03:28.766ZWrote out file "/tmp/test_repair_verify.data"
136222024-02-15T03:03:28.767Z
136232024-02-15T03:03:28.777ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136242024-02-15T03:03:28.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136252024-02-15T03:03:28.789Z 0 000-019 119 119 119 200 200 200 T T F <---
136262024-02-15T03:03:28.789Z 1 020-039 121 121 121 203 203 203 T T F <---
136272024-02-15T03:03:28.789Z 2 040-059 120 120 120 201 201 201 F F F
136282024-02-15T03:03:28.789Z 3 060-079 120 120 120 201 201 201 F F F
136292024-02-15T03:03:28.789Z 4 080-099 115 115 115 195 195 195 T T F <---
136302024-02-15T03:03:28.789Z 5 100-119 117 117 117 197 197 197 F F F
136312024-02-15T03:03:28.790Z 6 120-139 113 113 113 192 192 192 T T T
136322024-02-15T03:03:28.790Z 7 140-159 121 121 121 203 203 203 F F F
136332024-02-15T03:03:28.790Z 8 160-179 114 114 114 193 193 193 F F F
136342024-02-15T03:03:28.790Z 9 180-199 117 117 117 197 197 197 F F F
136352024-02-15T03:03:28.790Z 10 200-219 119 119 119 200 200 200 T T F <---
136362024-02-15T03:03:28.790Z 11 220-239 119 119 119 200 200 200 F F F
136372024-02-15T03:03:28.790Z 12 240-259 121 121 121 203 203 203 F F F
136382024-02-15T03:03:28.790Z 13 260-279 121 121 121 203 203 203 T T F <---
136392024-02-15T03:03:28.790Z 14 280-299 121 121 121 203 203 203 T T F <---
136402024-02-15T03:03:28.790Z 15 300-319 121 121 121 203 203 203 F F F
136412024-02-15T03:03:28.790Z 16 320-339 117 117 117 197 197 197 T T F <---
136422024-02-15T03:03:28.790Z 17 340-359 121 121 121 203 203 203 F F F
136432024-02-15T03:03:28.790Z 18 360-379 121 121 121 203 203 203 F F F
136442024-02-15T03:03:28.790Z 19 380-399 121 121 121 203 203 203 T T F <---
136452024-02-15T03:03:28.790Z 20 400-419 119 119 119 200 200 200 T T T
136462024-02-15T03:03:28.790Z 21 420-439 120 120 120 202 202 202 T T F <---
136472024-02-15T03:03:28.790Z 22 440-459 120 120 120 202 202 202 F F F
136482024-02-15T03:03:28.790Z 23 460-479 120 120 120 201 201 201 F F F
136492024-02-15T03:03:28.790Z 24 480-499 113 113 113 192 192 192 T T F <---
136502024-02-15T03:03:28.791Z 25 500-519 120 120 120 202 202 202 T T F <---
136512024-02-15T03:03:28.791Z 26 520-539 120 120 120 202 202 202 T T F <---
136522024-02-15T03:03:28.791Z 27 540-559 119 119 119 200 200 200 F F F
136532024-02-15T03:03:28.791Z 28 560-579 119 119 119 200 200 200 T T F <---
136542024-02-15T03:03:28.791Z 29 580-599 113 113 113 192 192 192 T T F <---
136552024-02-15T03:03:28.791ZMax gen: 121, Max flush: 203
136562024-02-15T03:03:28.791ZError: Difference in extent metadata found!
136572024-02-15T03:03:28.791ZA Difference in extent metadata is expected here
136582024-02-15T03:03:28.791ZOn loop 61
136592024-02-15T03:03:28.791Z
136602024-02-15T03:03:28.801ZVerifying data now
136612024-02-15T03:03:34.602ZNo change after verify
136622024-02-15T03:03:34.602ZLoop: 61 Downstairs dump after verify (and repair):
136632024-02-15T03:03:34.621ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136642024-02-15T03:03:34.621Z 0 000-019 122 122 122 204 204 204 F F F
136652024-02-15T03:03:34.621Z 1 020-039 122 122 122 204 204 204 F F F
136662024-02-15T03:03:34.621Z 2 040-059 120 120 120 201 201 201 F F F
136672024-02-15T03:03:34.621Z 3 060-079 120 120 120 201 201 201 F F F
136682024-02-15T03:03:34.621Z 4 080-099 122 122 122 204 204 204 F F F
136692024-02-15T03:03:34.621Z 5 100-119 117 117 117 197 197 197 F F F
136702024-02-15T03:03:34.621Z 6 120-139 122 122 122 204 204 204 F F F
136712024-02-15T03:03:34.621Z 7 140-159 121 121 121 203 203 203 F F F
136722024-02-15T03:03:34.621Z 8 160-179 114 114 114 193 193 193 F F F
136732024-02-15T03:03:34.622Z 9 180-199 117 117 117 197 197 197 F F F
136742024-02-15T03:03:34.622Z 10 200-219 122 122 122 204 204 204 F F F
136752024-02-15T03:03:34.622Z 11 220-239 119 119 119 200 200 200 F F F
136762024-02-15T03:03:34.622Z 12 240-259 121 121 121 203 203 203 F F F
136772024-02-15T03:03:34.633Z 13 260-279 122 122 122 204 204 204 F F F
136782024-02-15T03:03:34.633Z 14 280-299 122 122 122 204 204 204 F F F
136792024-02-15T03:03:34.633Z 15 300-319 121 121 121 203 203 203 F F F
136802024-02-15T03:03:34.633Z 16 320-339 122 122 122 204 204 204 F F F
136812024-02-15T03:03:34.634Z 17 340-359 121 121 121 203 203 203 F F F
136822024-02-15T03:03:34.634Z 18 360-379 121 121 121 203 203 203 F F F
136832024-02-15T03:03:34.634Z 19 380-399 122 122 122 204 204 204 F F F
136842024-02-15T03:03:34.634Z 20 400-419 122 122 122 204 204 204 F F F
136852024-02-15T03:03:34.634Z 21 420-439 122 122 122 204 204 204 F F F
136862024-02-15T03:03:34.634Z 22 440-459 120 120 120 202 202 202 F F F
136872024-02-15T03:03:34.634Z 23 460-479 120 120 120 201 201 201 F F F
136882024-02-15T03:03:34.634Z 24 480-499 122 122 122 204 204 204 F F F
136892024-02-15T03:03:34.634Z 25 500-519 122 122 122 204 204 204 F F F
136902024-02-15T03:03:34.634Z 26 520-539 122 122 122 204 204 204 F F F
136912024-02-15T03:03:34.634Z 27 540-559 119 119 119 200 200 200 F F F
136922024-02-15T03:03:34.634Z 28 560-579 122 122 122 204 204 204 F F F
136932024-02-15T03:03:34.634Z 29 580-599 122 122 122 204 204 204 F F F
136942024-02-15T03:03:34.634ZMax gen: 122, Max flush: 204
136952024-02-15T03:03:34.634Z
136962024-02-15T03:03:34.634ZBegin loop 62
136972024-02-15T03:03:34.634ZDownstairs to restart: 2
136982024-02-15T03:03:34.649Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.546979053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634}
136992024-02-15T03:03:34.649Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\nCrucible runtime is spawned
137002024-02-15T03:03:34.649Z rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547388251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634}
137012024-02-15T03:03:34.664Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547419687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634}
137022024-02-15T03:03:34.664Z{"msg":"Crucible 3a3e0f34-65a0-4c78-acc9-73cc89861322 has session id: 67a37f05-0e42-4d9f-913f-a2e98014386e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.54747526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137032024-02-15T03:03:34.664Z{"msg":"Upstairs opts: Upstairs UUID: 3a3e0f34-65a0-4c78-acc9-73cc89861322, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547544672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137042024-02-15T03:03:34.664Z{"msg":"Crucible stats registered with UUID: 3a3e0f34-65a0-4c78-acc9-73cc89861322","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547575548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137052024-02-15T03:03:34.664Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547602175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634}
137062024-02-15T03:03:34.664Z{"msg":"3a3e0f34-65a0-4c78-acc9-73cc89861322 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547701952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137072024-02-15T03:03:34.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547731008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137082024-02-15T03:03:34.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547747986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137092024-02-15T03:03:34.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547777043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137102024-02-15T03:03:34.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.5477981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137112024-02-15T03:03:34.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547817508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137122024-02-15T03:03:34.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547836375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137132024-02-15T03:03:34.665Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547852863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137142024-02-15T03:03:34.665Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547886999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137152024-02-15T03:03:34.665Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.547903257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137162024-02-15T03:03:34.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.548048439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137172024-02-15T03:03:34.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.548088064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137182024-02-15T03:03:34.665Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:33.548266181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137192024-02-15T03:03:34.665Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:33.548321214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137202024-02-15T03:03:34.665Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:33.548382587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137212024-02-15T03:03:34.665Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.548412883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137222024-02-15T03:03:34.665Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.548427601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137232024-02-15T03:03:34.665Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.548460667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137242024-02-15T03:03:34.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.548902212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137252024-02-15T03:03:34.665Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.548944076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137262024-02-15T03:03:34.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.54899213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137272024-02-15T03:03:34.665Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.549060112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137282024-02-15T03:03:34.665Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.549615882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137292024-02-15T03:03:34.665Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.549738067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137302024-02-15T03:03:34.665Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.550161134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137312024-02-15T03:03:34.665Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.550207238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137322024-02-15T03:03:34.665Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.550232365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137332024-02-15T03:03:34.665Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.550255672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137342024-02-15T03:03:34.666Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.550289268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137352024-02-15T03:03:34.666Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:33.550322994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137362024-02-15T03:03:35.648Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.550240747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137372024-02-15T03:03:35.648Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.550338235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137382024-02-15T03:03:35.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.55125722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137392024-02-15T03:03:35.666Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.551307223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137402024-02-15T03:03:35.667Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.551918597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137412024-02-15T03:03:35.667Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552514232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137422024-02-15T03:03:35.667Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552550268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137432024-02-15T03:03:35.667Z{"msg":"[0]R flush_numbers[0..12]: [204, 204, 201, 201, 204, 197, 204, 203, 193, 197, 204, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552567145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137442024-02-15T03:03:35.667Z{"msg":"[0]R generation[0..12]: [122, 122, 120, 120, 122, 117, 122, 121, 114, 117, 122, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552640107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137452024-02-15T03:03:35.667Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552658614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137462024-02-15T03:03:35.668Z{"msg":"[1]R flush_numbers[0..12]: [204, 204, 201, 201, 204, 197, 204, 203, 193, 197, 204, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.55269077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137472024-02-15T03:03:35.668Z{"msg":"[1]R generation[0..12]: [122, 122, 120, 120, 122, 117, 122, 121, 114, 117, 122, 119]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
137482024-02-15T03:03:35.668Z,"time":"2024-02-15T03:03:34.552710687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137492024-02-15T03:03:35.668Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552739634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137502024-02-15T03:03:35.668Z{"msg":"[2]R flush_numbers[0..12]: [204, 204, 201, 201, 204, 197, 204, 203, 193, 197, 204, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.55277258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137512024-02-15T03:03:35.668Z{"msg":"[2]R generation[0..12]: [122, 122, 120, 120, 122, 117, 122, 121, 114, 117, 122, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552812435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137522024-02-15T03:03:35.669Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552838451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137532024-02-15T03:03:35.669Z{"msg":"Max found gen is 123","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552861879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137542024-02-15T03:03:35.669Z{"msg":"Generation requested: 124 >= found:123","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552895164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137552024-02-15T03:03:35.669Z{"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552919571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137562024-02-15T03:03:35.669Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552950637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137572024-02-15T03:03:35.669Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.552974874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137582024-02-15T03:03:35.670Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.55300875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137592024-02-15T03:03:35.670Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.553034487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137602024-02-15T03:03:35.670Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.553060444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137612024-02-15T03:03:35.670Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.553085541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137622024-02-15T03:03:35.671Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.553111547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137632024-02-15T03:03:35.671Z{"msg":"3a3e0f34-65a0-4c78-acc9-73cc89861322 is now active with session: 67a37f05-0e42-4d9f-913f-a2e98014386e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.553125426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137642024-02-15T03:03:35.671Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.553140134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634,"session_id":"67a37f05-0e42-4d9f-913f-a2e98014386e"}
137652024-02-15T03:03:35.671Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:34.55317096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634}
137662024-02-15T03:03:35.671ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137672024-02-15T03:03:35.671ZLoading write count information from file "/tmp/test_repair_verify.data"
137682024-02-15T03:03:35.672ZRun Repair workload
137692024-02-15T03:03:35.672Z01/30 Write block 417 len 28672 data: 8 8 9 10 9 8 10
137702024-02-15T03:03:35.672Z02/30 Write block 496 len 36864 data: 13 13 13 11 11 11 12 13 12
137712024-02-15T03:03:35.672Z03/30 Write block 355 len 28672 data: 8 8 8 11 12 9 9
137722024-02-15T03:03:35.672Z04/30 Write block 33 len 32768 data: 9 9 10 9 9 10 8 6
137732024-02-15T03:03:35.672Z05/30 Read block 406 len 8192
137742024-02-15T03:03:35.672Z06/30 Write block 5 len 16384 data: 7 8 8 11
137752024-02-15T03:03:35.673Z07/30 Read block 477 len 8192
137762024-02-15T03:03:35.673Z08/30 Read block 313 len 24576
137772024-02-15T03:03:35.696Z09/30 Flush
137782024-02-15T03:03:35.697Z10/30 Write block 349 len 12288 data: 10 11 12
137792024-02-15T03:03:35.697Z11/30 Write block 144 len 12288 data: 11 10 10
137802024-02-15T03:03:35.697Z12/30 Write block 180 len 8192 data: 9 8
137812024-02-15T03:03:35.697Z13/30 Read block 402 len 36864
137822024-02-15T03:03:35.697Z14/30 Write block 110 len 8192 data: 5 6
137832024-02-15T03:03:35.697Z15/30 Read block 232 len 28672
137842024-02-15T03:03:35.697Z16/30 Read block 45 len 16384
137852024-02-15T03:03:35.698Z17/30 Write block 478 len 20480 data: 11 13 11 11 10
137862024-02-15T03:03:35.698Z18/30 Read block 558 len 12288
137872024-02-15T03:03:35.698Z19/30 Read block 395 len 28672
137882024-02-15T03:03:35.698Z20/30 Write block 123 len 40960 data: 7 7 6 7 7 6 6 6 7 9
137892024-02-15T03:03:35.698Z21/30 Write block 452 len 20480 data: 5 4 5 7 7
137902024-02-15T03:03:35.698Z22/30 Read block 357 len 20480
137912024-02-15T03:03:35.698Z23/30 Read block 382 len 4096
137922024-02-15T03:03:35.698Z24/30 Write block 387 len 32768 data: 13 13 14 12 11 10 9 10
137932024-02-15T03:03:35.698Z25/30 Write block 189 len 36864 data: 11 10 9 10 13 13 14 12 11
137942024-02-15T03:03:35.698Z26/30 Write block 126 len 40960 data: 8 8 7 7 7 8 10 11 10 10
137952024-02-15T03:03:35.698Z27/30 Read block 300 len 40960
137962024-02-15T03:03:35.698Z28/30 Read block 443 len 32768
137972024-02-15T03:03:35.698Z29/30 Read block 93 len 8192
137982024-02-15T03:03:35.699Z30/30 Read block 484 len 36864
137992024-02-15T03:03:35.699Z----------------------------------------------------------------
138002024-02-15T03:03:35.699Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
138012024-02-15T03:03:35.699ZStates: Active Active Active
138022024-02-15T03:03:35.699ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138032024-02-15T03:03:35.699Z 1 Acked 1000 Write 7 Done Done Sent false
138042024-02-15T03:03:35.699Z 2 Acked 1001 Write 9 Done Done Sent false
138052024-02-15T03:03:35.699Z 3 Acked 1002 Write 7 Done Done Sent false
138062024-02-15T03:03:35.700Z 4 Acked 1003 Write 8 Done Done Sent false
138072024-02-15T03:03:35.700Z 5 Acked 1004 Read 2 Done Done Sent false
138082024-02-15T03:03:35.700Z 6 Acked 1005 Write 4 Done Done Sent false
138092024-02-15T03:03:35.700Z 7 Acked 1006 Read 2 Done Done Sent false
138102024-02-15T03:03:35.700Z 8 Acked 1007 Read 6 Done Done Sent false
138112024-02-15T03:03:35.700Z 9 Acked 1008 Flush 0 Done Done Sent false
138122024-02-15T03:03:35.700Z 10 Acked 1009 Write 3 Done Done Sent false
138132024-02-15T03:03:35.700Z 11 Acked 1010 Write 3 Done Done Sent false
138142024-02-15T03:03:35.700Z 12 Acked 1011 Write 2 Done Done Sent false
138152024-02-15T03:03:35.700Z 13 Acked 1012 Read 9 Done Done Sent false
138162024-02-15T03:03:35.701Z 14 Acked 1013 Write 2 Done Done Sent false
138172024-02-15T03:03:35.701Z 15 Acked 1014 Read 7 Done Done Sent false
138182024-02-15T03:03:35.701Z 16 Acked 1015 Read 4 Done Done Sent false
138192024-02-15T03:03:35.701Z 17 Acked 1016 Write 5 Done Done Sent false
138202024-02-15T03:03:35.701Z 18 Acked 1017 Read 3 Done Done Sent false
138212024-02-15T03:03:35.701Z 19 Acked 1018 Read 7 Done Done Sent false
138222024-02-15T03:03:35.701Z 20 Acked 1019 Write 10 Done Done Sent false
138232024-02-15T03:03:35.701Z 21 Acked 1020 Write 5 Done Done Sent false
138242024-02-15T03:03:35.701Z 22 Acked 1021 Read 5 Done Done Sent false
138252024-02-15T03:03:35.702Z 23 Acked 1022 Read 1 Done Done Sent false
138262024-02-15T03:03:35.702Z 24 Acked 1023 Write 8 Done Done Sent false
138272024-02-15T03:03:35.702Z 25 Acked 1024 Write 9 Done Done Sent false
138282024-02-15T03:03:35.702Z 26 Acked 1025 Write 10 Done Done Sent false
138292024-02-15T03:03:35.702Z 27 Acked 1026 Read 10 Done Done Sent false
138302024-02-15T03:03:35.702Z 28 Acked 1027 Read 8 Done Done Sent false
138312024-02-15T03:03:35.702Z 29 Acked 1028 Read 2 Done Done Sent false
138322024-02-15T03:03:35.702Z 30 Acked 1029 Read 9 Done Done Sent false
138332024-02-15T03:03:35.702Z STATES DS:0 DS:1 DS:2 TOTAL
138342024-02-15T03:03:35.703Z New 0 0 0 0
138352024-02-15T03:03:35.703Z Sent 0 0 30 30
138362024-02-15T03:03:35.703Z Done 30 30 0 60
138372024-02-15T03:03:35.703Z Skipped 0 0 0 0
138382024-02-15T03:03:35.703Z Error 0 0 0 0
138392024-02-15T03:03:35.703ZLast Flush: 1008 1008 0
138402024-02-15T03:03:35.703ZDownstairs last five completed:
138412024-02-15T03:03:35.703ZUpstairs last five completed: 30 29 28 27 26
138422024-02-15T03:03:35.703Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:34.584031343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1634}
138432024-02-15T03:03:35.703ZWrote out file "/tmp/test_repair_verify.data"
138442024-02-15T03:03:35.703Z
138452024-02-15T03:03:35.708ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138462024-02-15T03:03:35.708ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138472024-02-15T03:03:35.708Z 0 000-019 124 124 122 205 205 204 F F F <---
138482024-02-15T03:03:35.708Z 1 020-039 124 124 122 205 205 204 F F F <---
138492024-02-15T03:03:35.708Z 2 040-059 124 124 120 205 205 201 F F F <---
138502024-02-15T03:03:35.708Z 3 060-079 120 120 120 201 201 201 F F F
138512024-02-15T03:03:35.708Z 4 080-099 122 122 122 204 204 204 F F F
138522024-02-15T03:03:35.708Z 5 100-119 117 117 117 197 197 197 T T F <---
138532024-02-15T03:03:35.708Z 6 120-139 122 122 122 204 204 204 T T F <---
138542024-02-15T03:03:35.708Z 7 140-159 121 121 121 203 203 203 T T F <---
138552024-02-15T03:03:35.708Z 8 160-179 114 114 114 193 193 193 F F F
138562024-02-15T03:03:35.708Z 9 180-199 117 117 117 197 197 197 T T F <---
138572024-02-15T03:03:35.708Z 10 200-219 122 122 122 204 204 204 F F F
138582024-02-15T03:03:35.708Z 11 220-239 119 119 119 200 200 200 F F F
138592024-02-15T03:03:35.708Z 12 240-259 121 121 121 203 203 203 F F F
138602024-02-15T03:03:35.708Z 13 260-279 122 122 122 204 204 204 F F F
138612024-02-15T03:03:35.708Z 14 280-299 122 122 122 204 204 204 F F F
138622024-02-15T03:03:35.709Z 15 300-319 121 121 121 203 203 203 F F F
138632024-02-15T03:03:35.709Z 16 320-339 122 122 122 204 204 204 F F F
138642024-02-15T03:03:35.709Z 17 340-359 124 124 121 205 205 203 T T F <---
138652024-02-15T03:03:35.709Z 18 360-379 124 124 121 205 205 203 F F F <---
138662024-02-15T03:03:35.709Z 19 380-399 122 122 122 204 204 204 T T F <---
138672024-02-15T03:03:35.709Z 20 400-419 124 124 122 205 205 204 F F F <---
138682024-02-15T03:03:35.709Z 21 420-439 124 124 122 205 205 204 F F F <---
138692024-02-15T03:03:35.709Z 22 440-459 120 120 120 202 202 202 T T F <---
138702024-02-15T03:03:35.709Z 23 460-479 120 120 120 201 201 201 T T F <---
138712024-02-15T03:03:35.709Z 24 480-499 124 124 122 205 205 204 T T F <---
138722024-02-15T03:03:35.709Z 25 500-519 124 124 122 205 205 204 F F F <---
138732024-02-15T03:03:35.709Z 26 520-539 122 122 122 204 204 204 F F F
138742024-02-15T03:03:35.709Z 27 540-559 119 119 119 200 200 200 F F F
138752024-02-15T03:03:35.709Z 28 560-579 122 122 122 204 204 204 F F F
138762024-02-15T03:03:35.709Z 29 580-599 122 122 122 204 204 204 F F F
138772024-02-15T03:03:35.709ZMax gen: 124, Max flush: 205
138782024-02-15T03:03:35.712ZError: Difference in extent metadata found!
138792024-02-15T03:03:35.712ZA Difference in extent metadata is expected here
138802024-02-15T03:03:35.712ZOn loop 62
138812024-02-15T03:03:35.713Z
138822024-02-15T03:03:35.716ZVerifying data now
138832024-02-15T03:03:40.511ZNo change after verify
138842024-02-15T03:03:40.511ZLoop: 62 Downstairs dump after verify (and repair):
138852024-02-15T03:03:40.527ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138862024-02-15T03:03:40.527Z 0 000-019 124 124 124 205 205 205 F F F
138872024-02-15T03:03:40.527Z 1 020-039 124 124 124 205 205 205 F F F
138882024-02-15T03:03:40.527Z 2 040-059 124 124 124 205 205 205 F F F
138892024-02-15T03:03:40.527Z 3 060-079 120 120 120 201 201 201 F F F
138902024-02-15T03:03:40.527Z 4 080-099 122 122 122 204 204 204 F F F
138912024-02-15T03:03:40.528Z 5 100-119 125 125 125 206 206 206 F F F
138922024-02-15T03:03:40.528Z 6 120-139 125 125 125 206 206 206 F F F
138932024-02-15T03:03:40.532Z 7 140-159 125 125 125 206 206 206 F F F
138942024-02-15T03:03:40.532Z 8 160-179 114 114 114 193 193 193 F F F
138952024-02-15T03:03:40.533Z 9 180-199 125 125 125 206 206 206 F F F
138962024-02-15T03:03:40.533Z 10 200-219 122 122 122 204 204 204 F F F
138972024-02-15T03:03:40.533Z 11 220-239 119 119 119 200 200 200 F F F
138982024-02-15T03:03:40.533Z 12 240-259 121 121 121 203 203 203 F F F
138992024-02-15T03:03:40.533Z 13 260-279 122 122 122 204 204 204 F F F
139002024-02-15T03:03:40.533Z 14 280-299 122 122 122 204 204 204 F F F
139012024-02-15T03:03:40.533Z 15 300-319 121 121 121 203 203 203 F F F
139022024-02-15T03:03:40.534Z 16 320-339 122 122 122 204 204 204 F F F
139032024-02-15T03:03:40.534Z 17 340-359 125 125 125 206 206 206 F F F
139042024-02-15T03:03:40.534Z 18 360-379 124 124 124 205 205 205 F F F
139052024-02-15T03:03:40.534Z 19 380-399 125 125 125 206 206 206 F F F
139062024-02-15T03:03:40.534Z 20 400-419 124 124 124 205 205 205 F F F
139072024-02-15T03:03:40.534Z 21 420-439 124 124 124 205 205 205 F F F
139082024-02-15T03:03:40.535Z 22 440-459 125 125 125 206 206 206 F F F
139092024-02-15T03:03:40.535Z 23 460-479 125 125 125 206 206 206 F F F
139102024-02-15T03:03:40.535Z 24 480-499 125 125 125 206 206 206 F F F
139112024-02-15T03:03:40.535Z 25 500-519 124 124 124 205 205 205 F F F
139122024-02-15T03:03:40.535Z 26 520-539 122 122 122 204 204 204 F F F
139132024-02-15T03:03:40.536Z 27 540-559 119 119 119 200 200 200 F F F
139142024-02-15T03:03:40.536Z 28 560-579 122 122 122 204 204 204 F F F
139152024-02-15T03:03:40.536Z 29 580-599 122 122 122 204 204 204 F F F
139162024-02-15T03:03:40.536ZMax gen: 125, Max flush: 206
139172024-02-15T03:03:40.536Z
139182024-02-15T03:03:40.536ZBegin loop 63
139192024-02-15T03:03:40.536ZDownstairs to restart: 2
139202024-02-15T03:03:40.550Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.453573184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642}
139212024-02-15T03:03:40.550Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.453830902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642}
139222024-02-15T03:03:40.558Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.453857268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642}
139232024-02-15T03:03:40.558Z{"msg":"Crucible 856db75b-8e15-41ba-9fcd-c423d5830e2e has session id: 88ad502a-a789-42a6-b495-c88acd49a697","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.453887675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139242024-02-15T03:03:40.558Z{"msg":"Upstairs opts: Upstairs UUID: 856db75b-8e15-41ba-9fcd-c423d5830e2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.45392675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139252024-02-15T03:03:40.558Z{"msg":"Crucible stats registered with UUID: 856db75b-8e15-41ba-9fcd-c423d5830e2e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.453966315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139262024-02-15T03:03:40.558ZCrucible runtime is spawned
139272024-02-15T03:03:40.558Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454145752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642}
139282024-02-15T03:03:40.558Z{"msg":"856db75b-8e15-41ba-9fcd-c423d5830e2e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.4542383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139292024-02-15T03:03:40.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454292064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139302024-02-15T03:03:40.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454332889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139312024-02-15T03:03:40.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454364865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139322024-02-15T03:03:40.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454414888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139332024-02-15T03:03:40.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454460523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139342024-02-15T03:03:40.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454508337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139352024-02-15T03:03:40.559Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454550381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139362024-02-15T03:03:40.560Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454613884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139372024-02-15T03:03:40.560Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.45464346Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139382024-02-15T03:03:40.560Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:39.454685944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139392024-02-15T03:03:40.560Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:39.454727159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139402024-02-15T03:03:40.560Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:39.454758485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139412024-02-15T03:03:40.561Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454780923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139422024-02-15T03:03:40.561Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454820418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139432024-02-15T03:03:40.561Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454865702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139442024-02-15T03:03:40.561Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454892489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139452024-02-15T03:03:40.561Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.454938943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139462024-02-15T03:03:40.561Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.455374258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139472024-02-15T03:03:40.562Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.455412064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139482024-02-15T03:03:40.562Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.455430901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139492024-02-15T03:03:40.562Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.455562045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139502024-02-15T03:03:40.562Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456007079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139512024-02-15T03:03:40.562Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456133613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139522024-02-15T03:03:40.562Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456515245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139532024-02-15T03:03:40.562Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456552271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139542024-02-15T03:03:40.563Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456610323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139552024-02-15T03:03:40.563Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456666056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139562024-02-15T03:03:40.563Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456711031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139572024-02-15T03:03:40.563Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:39.456725779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139582024-02-15T03:03:41.600Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.456282458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139592024-02-15T03:03:41.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.456364467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139602024-02-15T03:03:41.615Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.457232829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139612024-02-15T03:03:41.615Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.457277463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139622024-02-15T03:03:41.616Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.457773821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139632024-02-15T03:03:41.616Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.45833906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139642024-02-15T03:03:41.616Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458375456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139652024-02-15T03:03:41.616Z{"msg":"[0]R flush_numbers[0..12]: [205, 205, 205, 201, 204, 206, 206, 206, 193, 206, 204, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458391223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139662024-02-15T03:03:41.616Z{"msg":"[0]R generation[0..12]: [124, 124, 124, 120, 122, 125, 125, 125, 114, 125, 122, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458409831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139672024-02-15T03:03:41.616Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458445857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139682024-02-15T03:03:41.616Z{"msg":"[1]R flush_numbers[0..12]: [205, 205, 205, 201, 204, 206, 206, 206, 193, 206, 204, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458476093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139692024-02-15T03:03:41.617Z{"msg":"[1]R generation[0..12]: [124, 124, 124, 120, 122, 125, 125, 125, 114, 125, 122, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.45849903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139702024-02-15T03:03:41.617Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458513858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139712024-02-15T03:03:41.617Z{"msg":"[2]R flush_numbers[0..12]: [205, 205, 205, 201, 204, 206, 206, 206, 193, 206, 204, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458535025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139722024-02-15T03:03:41.617Z{"msg":"[2]R generation[0..12]: [124, 124, 124, 120, 122, 125, 125, 125, 114, 125, 122, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458549104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139732024-02-15T03:03:41.617Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458571451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139742024-02-15T03:03:41.618Z{"msg":"Max found gen is 126Wait for a query_work_queue command to finish before sending IO
139752024-02-15T03:03:41.618Z","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458588579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139762024-02-15T03:03:41.618Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458609236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139772024-02-15T03:03:41.618Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458622665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139782024-02-15T03:03:41.618Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458642402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139792024-02-15T03:03:41.619Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.4586562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139802024-02-15T03:03:41.619Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458675058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139812024-02-15T03:03:41.619Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458688726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139822024-02-15T03:03:41.619Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458708634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139832024-02-15T03:03:41.619Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458722202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139842024-02-15T03:03:41.619Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458746709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139852024-02-15T03:03:41.620Z{"msg":"856db75b-8e15-41ba-9fcd-c423d5830e2e is now active with session: 88ad502a-a789-42a6-b495-c88acd49a697","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458760447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139862024-02-15T03:03:41.620Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458780055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642,"session_id":"88ad502a-a789-42a6-b495-c88acd49a697"}
139872024-02-15T03:03:41.620Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:40.458793073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642}
139882024-02-15T03:03:41.620ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
139892024-02-15T03:03:41.620ZLoading write count information from file "/tmp/test_repair_verify.data"
139902024-02-15T03:03:41.620ZRun Repair workload
139912024-02-15T03:03:41.620Z01/30 Write block 469 len 28672 data: 11 13 11 9 8 7 6
139922024-02-15T03:03:41.621Z02/30 Flush
139932024-02-15T03:03:41.621Z03/30 Write block 186 len 8192 data: 11 8
139942024-02-15T03:03:41.621Z04/30 Write block 326 len 24576 data: 11 10 10 11 12 13
139952024-02-15T03:03:41.621Z05/30 Write block 35 len 36864 data: 11 10 10 11 9 7 8 9 8
139962024-02-15T03:03:41.621Z06/30 Write block 175 len 20480 data: 6 5 7 8 9
139972024-02-15T03:03:41.621Z07/30 Flush
139982024-02-15T03:03:41.621Z08/30 Write block 353 len 40960 data: 11 9 9 9 9 12 13 10 10 10
139992024-02-15T03:03:41.622Z09/30 Read block 385 len 20480
140002024-02-15T03:03:41.622Z10/30 Flush
140012024-02-15T03:03:41.622Z11/30 Write block 246 len 24576 data: 10 10 8 9 9 9
140022024-02-15T03:03:41.622Z12/30 Read block 375 len 20480
140032024-02-15T03:03:41.622Z13/30 Read block 467 len 36864
140042024-02-15T03:03:41.622Z14/30 Read block 542 len 32768
140052024-02-15T03:03:41.622Z15/30 Read block 402 len 28672
140062024-02-15T03:03:41.622Z16/30 Read block 479 len 4096
140072024-02-15T03:03:41.622Z17/30 Read block 290 len 16384
140082024-02-15T03:03:41.623Z18/30 Read block 549 len 4096
140092024-02-15T03:03:41.623Z19/30 Write block 98 len 28672 data: 14 12 11 12 12 10 12
140102024-02-15T03:03:41.623Z20/30 Write block 236 len 8192 data: 13 11
140112024-02-15T03:03:41.623Z21/30 Read block 205 len 4096
140122024-02-15T03:03:41.623Z22/30 Read block 26 len 16384
140132024-02-15T03:03:41.623Z23/30 Read block 243 len 40960
140142024-02-15T03:03:41.623Z24/30 Read block 217 len 24576
140152024-02-15T03:03:41.623Z25/30 Write block 354 len 16384 data: 10 10 10 10
140162024-02-15T03:03:41.623Z26/30 Read block 236 len 28672
140172024-02-15T03:03:41.623Z27/30 Read block 356 len 20480
140182024-02-15T03:03:41.624Z28/30 Read block 417 len 28672
140192024-02-15T03:03:41.624Z29/30 Read block 153 len 16384
140202024-02-15T03:03:41.624Z30/30 Write block 270 len 4096 data: 10
140212024-02-15T03:03:41.624Z----------------------------------------------------------------
140222024-02-15T03:03:41.624Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:28
140232024-02-15T03:03:41.624ZStates: Active Active Active
140242024-02-15T03:03:41.624ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140252024-02-15T03:03:41.624Z 3 Acked 1002 Write 2 Done Done Done false
140262024-02-15T03:03:41.624Z 4 Acked 1003 Write 6 Done Done Sent false
140272024-02-15T03:03:41.624Z 5 Acked 1004 Write 9 Done Done Sent false
140282024-02-15T03:03:41.625Z 6 Acked 1005 Write 5 Done Done Sent false
140292024-02-15T03:03:41.625Z 7 Acked 1006 Flush 0 Done Done Sent false
140302024-02-15T03:03:41.625Z 8 Acked 1007 Write 10 Done Done Sent false
140312024-02-15T03:03:41.625Z 9 Acked 1008 Read 5 Done Done Sent false
140322024-02-15T03:03:41.625Z 10 Acked 1009 Flush 0 Done Done Sent false
140332024-02-15T03:03:41.625Z 11 Acked 1010 Write 6 Done Done Sent false
140342024-02-15T03:03:41.625Z 12 Acked 1011 Read 5 Done Done Sent false
140352024-02-15T03:03:41.626Z 13 Acked 1012 Read 9 Done Done Sent false
140362024-02-15T03:03:41.626Z 14 Acked 1013 Read 8 Done Done Sent false
140372024-02-15T03:03:41.626Z 15 Acked 1014 Read 7 Done Done Sent false
140382024-02-15T03:03:41.626Z 16 Acked 1015 Read 1 Done Done Sent false
140392024-02-15T03:03:41.626Z 17 Acked 1016 Read 4 Done Done Sent false
140402024-02-15T03:03:41.626Z 18 Acked 1017 Read 1 Done Done Sent false
140412024-02-15T03:03:41.627Z 19 Acked 1018 Write 7 Done Done Sent false
140422024-02-15T03:03:41.627Z 20 Acked 1019 Write 2 Done Done Sent false
140432024-02-15T03:03:41.627Z 21 Acked 1020 Read 1 Done Done Sent false
140442024-02-15T03:03:41.627Z 22 Acked 1021 Read 4 Done Done Sent false
140452024-02-15T03:03:41.627Z 23 Acked 1022 Read 10 Done Done Sent false
140462024-02-15T03:03:41.627Z 24 Acked 1023 Read 6 Done Done Sent false
140472024-02-15T03:03:41.627Z 25 Acked 1024 Write 4 Done Done Sent false
140482024-02-15T03:03:41.627Z 26 Acked 1025 Read 7 Done Done Sent false
140492024-02-15T03:03:41.628Z 27 Acked 1026 Read 5 Done Done Sent false
140502024-02-15T03:03:41.628Z 28 Acked 1027 Read 7 Done Done Sent false
140512024-02-15T03:03:41.628Z 29 Acked 1028 Read 4 Done Done Sent false
140522024-02-15T03:03:41.628Z 30 Acked 1029 Write 1 Sent Sent Sent false
140532024-02-15T03:03:41.628Z STATES DS:0 DS:1 DS:2 TOTAL
140542024-02-15T03:03:41.628Z New 0 0 0 0
140552024-02-15T03:03:41.629Z Sent 1 1 27 29
140562024-02-15T03:03:41.629Z Done 27 27 1 55
140572024-02-15T03:03:41.629Z Skipped 0 0 0 0
140582024-02-15T03:03:41.629Z Error 0 0 0 0
140592024-02-15T03:03:41.629ZLast Flush: 1009 1009 1001
140602024-02-15T03:03:41.658ZDownstairs last five completed: 1001 1000
140612024-02-15T03:03:41.658ZUpstairs last five completed: 30 29 28 27 26
140622024-02-15T03:03:41.659Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:40.491801167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1642}
140632024-02-15T03:03:41.659ZWrote out file "/tmp/test_repair_verify.data"
140642024-02-15T03:03:41.659Z
140652024-02-15T03:03:41.659ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140662024-02-15T03:03:41.659ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140672024-02-15T03:03:41.659Z 0 000-019 124 124 124 205 205 205 F F F
140682024-02-15T03:03:41.660Z 1 020-039 126 126 124 208 208 205 F F F <---
140692024-02-15T03:03:41.660Z 2 040-059 126 126 124 208 208 205 F F F <---
140702024-02-15T03:03:41.660Z 3 060-079 120 120 120 201 201 201 F F F
140712024-02-15T03:03:41.660Z 4 080-099 122 122 122 204 204 204 T T F <---
140722024-02-15T03:03:41.660Z 5 100-119 125 125 125 206 206 206 T T F <---
140732024-02-15T03:03:41.660Z 6 120-139 125 125 125 206 206 206 F F F
140742024-02-15T03:03:41.660Z 7 140-159 125 125 125 206 206 206 F F F
140752024-02-15T03:03:41.661Z 8 160-179 126 126 114 208 208 193 F F F <---
140762024-02-15T03:03:41.661Z 9 180-199 126 126 125 208 208 206 F F T <---
140772024-02-15T03:03:41.661Z 10 200-219 122 122 122 204 204 204 F F F
140782024-02-15T03:03:41.662Z 11 220-239 119 119 119 200 200 200 T T F <---
140792024-02-15T03:03:41.662Z 12 240-259 121 121 121 203 203 203 T T F <---
140802024-02-15T03:03:41.662Z 13 260-279 122 122 122 204 204 204 T T F <---
140812024-02-15T03:03:41.662Z 14 280-299 122 122 122 204 204 204 F F F
140822024-02-15T03:03:41.663Z 15 300-319 121 121 121 203 203 203 F F F
140832024-02-15T03:03:41.663Z 16 320-339 126 126 122 208 208 204 F F F <---
140842024-02-15T03:03:41.663Z 17 340-359 126 126 125 209 209 206 T T F <---
140852024-02-15T03:03:41.663Z 18 360-379 126 126 124 209 209 205 F F F <---
140862024-02-15T03:03:41.663Z 19 380-399 125 125 125 206 206 206 F F F
140872024-02-15T03:03:41.664Z 20 400-419 124 124 124 205 205 205 F F F
140882024-02-15T03:03:41.664Z 21 420-439 124 124 124 205 205 205 F F F
140892024-02-15T03:03:41.664Z 22 440-459 125 125 125 206 206 206 F F F
140902024-02-15T03:03:41.664Z 23 460-479 126 126 126 207 207 207 F F F
140912024-02-15T03:03:41.664Z 24 480-499 125 125 125 206 206 206 F F F
140922024-02-15T03:03:41.664Z 25 500-519 124 124 124 205 205 205 F F F
140932024-02-15T03:03:41.664Z 26 520-539 122 122 122 204 204 204 F F F
140942024-02-15T03:03:41.664Z 27 540-559 119 119 119 200 200 200 F F F
140952024-02-15T03:03:41.665Z 28 560-579 122 122 122 204 204 204 F F F
140962024-02-15T03:03:41.665Z 29 580-599 122 122 122 204 204 204 F F F
140972024-02-15T03:03:41.665ZMax gen: 126, Max flush: 209
140982024-02-15T03:03:41.665ZError: Difference in extent metadata found!
140992024-02-15T03:03:41.665ZA Difference in extent metadata is expected here
141002024-02-15T03:03:41.665ZOn loop 63
141012024-02-15T03:03:41.665Z
141022024-02-15T03:03:41.665ZVerifying data now
141032024-02-15T03:03:47.251ZNo change after verify
141042024-02-15T03:03:47.251ZLoop: 63 Downstairs dump after verify (and repair):
141052024-02-15T03:03:47.702ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141062024-02-15T03:03:47.702Z 0 000-019 124 124 124 205 205 205 F F F
141072024-02-15T03:03:47.702Z 1 020-039 126 126 126 208 208 208 F F F
141082024-02-15T03:03:47.703Z 2 040-059 126 126 126 208 208 208 F F F
141092024-02-15T03:03:47.703Z 3 060-079 120 120 120 201 201 201 F F F
141102024-02-15T03:03:47.703Z 4 080-099 127 127 127 210 210 210 F F F
141112024-02-15T03:03:47.703Z 5 100-119 127 127 127 210 210 210 F F F
141122024-02-15T03:03:47.703Z 6 120-139 125 125 125 206 206 206 F F F
141132024-02-15T03:03:47.703Z 7 140-159 125 125 125 206 206 206 F F F
141142024-02-15T03:03:47.703Z 8 160-179 126 126 126 208 208 208 F F F
141152024-02-15T03:03:47.703Z 9 180-199 126 126 126 208 208 208 F F F
141162024-02-15T03:03:47.703Z 10 200-219 122 122 122 204 204 204 F F F
141172024-02-15T03:03:47.703Z 11 220-239 127 127 127 210 210 210 F F F
141182024-02-15T03:03:47.703Z 12 240-259 127 127 127 210 210 210 F F F
141192024-02-15T03:03:47.703Z 13 260-279 127 127 127 210 210 210 F F F
141202024-02-15T03:03:47.703Z 14 280-299 122 122 122 204 204 204 F F F
141212024-02-15T03:03:47.703Z 15 300-319 121 121 121 203 203 203 F F F
141222024-02-15T03:03:47.703Z 16 320-339 126 126 126 208 208 208 F F F
141232024-02-15T03:03:47.703Z 17 340-359 127 127 127 210 210 210 F F F
141242024-02-15T03:03:47.703Z 18 360-379 126 126 126 209 209 209 F F F
141252024-02-15T03:03:47.703Z 19 380-399 125 125 125 206 206 206 F F F
141262024-02-15T03:03:47.703Z 20 400-419 124 124 124 205 205 205 F F F
141272024-02-15T03:03:47.703Z 21 420-439 124 124 124 205 205 205 F F F
141282024-02-15T03:03:47.703Z 22 440-459 125 125 125 206 206 206 F F F
141292024-02-15T03:03:47.703Z 23 460-479 126 126 126 207 207 207 F F F
141302024-02-15T03:03:47.703Z 24 480-499 125 125 125 206 206 206 F F F
141312024-02-15T03:03:47.703Z 25 500-519 124 124 124 205 205 205 F F F
141322024-02-15T03:03:47.703Z 26 520-539 122 122 122 204 204 204 F F F
141332024-02-15T03:03:47.704Z 27 540-559 119 119 119 200 200 200 F F F
141342024-02-15T03:03:47.704Z 28 560-579 122 122 122 204 204 204 F F F
141352024-02-15T03:03:47.704Z 29 580-599 122 122 122 204 204 204 F F F
141362024-02-15T03:03:47.704ZMax gen: 127, Max flush: 210
141372024-02-15T03:03:47.704Z
141382024-02-15T03:03:47.704ZBegin loop 64
141392024-02-15T03:03:47.704ZDownstairs to restart: 1
141402024-02-15T03:03:47.704Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.187958446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650}
141412024-02-15T03:03:47.704Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
141422024-02-15T03:03:47.704Z,"time":"2024-02-15T03:03:46.188288665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650}
141432024-02-15T03:03:47.704Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.18832592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650}
141442024-02-15T03:03:47.704Z{"msg":"Crucible 0b05f64d-dada-4a20-b9f9-5129f40c911f has session id: 792fbb41-237a-48ba-8458-47fe2e06b29e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188382713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141452024-02-15T03:03:47.704Z{"msg":"Upstairs opts: Upstairs UUID: 0b05f64d-dada-4a20-b9f9-5129f40c911f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188433197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141462024-02-15T03:03:47.704Z{"msg":"Crucible stats registered with UUID: 0b05f64d-dada-4a20-b9f9-5129f40c911f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188468092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141472024-02-15T03:03:47.704Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.18848559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650}
141482024-02-15T03:03:47.704Z{"msg":"0b05f64d-dada-4a20-b9f9-5129f40c911f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188577788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141492024-02-15T03:03:47.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188609304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141502024-02-15T03:03:47.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188652039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141512024-02-15T03:03:47.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188694024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141522024-02-15T03:03:47.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188741928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141532024-02-15T03:03:47.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188788332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141542024-02-15T03:03:47.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188828957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141552024-02-15T03:03:47.704Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188865432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141562024-02-15T03:03:47.705Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188916716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141572024-02-15T03:03:47.705Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.188956271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141582024-02-15T03:03:47.705Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:46.189052829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141592024-02-15T03:03:47.705Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:46.189131589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141602024-02-15T03:03:47.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189190882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141612024-02-15T03:03:47.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189233746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141622024-02-15T03:03:47.705Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:46.189257993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141632024-02-15T03:03:47.705Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189297178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141642024-02-15T03:03:47.705Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189323005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141652024-02-15T03:03:47.705Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189353611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141662024-02-15T03:03:47.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189808994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141672024-02-15T03:03:47.705Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189848169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141682024-02-15T03:03:47.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189931849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141692024-02-15T03:03:47.705Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.189971224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141702024-02-15T03:03:47.705Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.190444354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141712024-02-15T03:03:47.705Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.190583337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141722024-02-15T03:03:47.705Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.190984357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141732024-02-15T03:03:47.705Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.191022892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141742024-02-15T03:03:47.705Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.191061837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141752024-02-15T03:03:47.705Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.191184082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141762024-02-15T03:03:47.705Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.191236225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141772024-02-15T03:03:47.705Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:46.191270231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141782024-02-15T03:03:48.296Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.191111244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141792024-02-15T03:03:48.726Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.191328577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141802024-02-15T03:03:48.726Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.192332351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141812024-02-15T03:03:48.726Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.192371826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141822024-02-15T03:03:48.726Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.192850606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141832024-02-15T03:03:48.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193418785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141842024-02-15T03:03:48.727Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193466319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141852024-02-15T03:03:48.727Z{"msg":"[0]R flush_numbers[0..12]: [205, 208, 208, 201, 210, 210, 206, 206, 208, 208, 204, 210]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193498125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141862024-02-15T03:03:48.727Z{"msg":"[0]R generation[0..12]: [124, 126, 126, 120, 127, 127, 125, 125, 126, 126, 122, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193549088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141872024-02-15T03:03:48.727Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193588354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141882024-02-15T03:03:48.727Z{"msg":"[1]R flush_numbers[0..12]: [205, 208, 208, 201, 210, 210, 206, 206, 208, 208, 204, 210]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193626799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141892024-02-15T03:03:48.727ZWait for a query_work_queue command to finish before sending IO
141902024-02-15T03:03:48.727Z{"msg":"[1]R generation[0..12]: [124, 126, 126, 120, 127, 127, 125, 125, 126, 126, 122, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193671943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141912024-02-15T03:03:48.727Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193713568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141922024-02-15T03:03:48.727Z{"msg":"[2]R flush_numbers[0..12]: [205, 208, 208, 201, 210, 210, 206, 206, 208, 208, 204, 210]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193729826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141932024-02-15T03:03:48.727Z{"msg":"[2]R generation[0..12]: [124, 126, 126, 120, 127, 127, 125, 125, 126, 126, 122, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193761382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141942024-02-15T03:03:48.727Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193795298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141952024-02-15T03:03:48.727Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193826944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141962024-02-15T03:03:48.727Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.19385812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141972024-02-15T03:03:48.727Z{"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193890646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141982024-02-15T03:03:48.727Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193921072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
141992024-02-15T03:03:48.728Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193953218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142002024-02-15T03:03:48.728Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.193983124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142012024-02-15T03:03:48.728Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.19401592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142022024-02-15T03:03:48.728Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142032024-02-15T03:03:48.728Z,"time":"2024-02-15T03:03:47.194040397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142042024-02-15T03:03:48.728Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.194075953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142052024-02-15T03:03:48.728Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.194100199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142062024-02-15T03:03:48.728Z{"msg":"0b05f64d-dada-4a20-b9f9-5129f40c911f is now active with session: 792fbb41-237a-48ba-8458-47fe2e06b29e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.194113978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142072024-02-15T03:03:48.728Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.194132935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650,"session_id":"792fbb41-237a-48ba-8458-47fe2e06b29e"}
142082024-02-15T03:03:48.728Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:47.194145654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650}
142092024-02-15T03:03:48.728ZLoading write count information from file "/tmp/test_repair_verify.data"
142102024-02-15T03:03:48.728ZRun Repair workload
142112024-02-15T03:03:48.728Z01/30 Write block 66 len 8192 data: 8 8
142122024-02-15T03:03:48.728Z02/30 Write block 587 len 20480 data: 9 8 12 13 15
142132024-02-15T03:03:48.728Z03/30 Read block 207 len 8192
142142024-02-15T03:03:48.728Z04/30 Read block 247 len 16384
142152024-02-15T03:03:48.728Z05/30 Write block 314 len 16384 data: 10 9 9 10
142162024-02-15T03:03:48.728Z06/30 Flush
142172024-02-15T03:03:48.728Z07/30 Write block 592 len 4096 data: 12
142182024-02-15T03:03:48.728Z08/30 Read block 576 len 12288
142192024-02-15T03:03:48.728Z09/30 Read block 10 len 40960
142202024-02-15T03:03:48.728Z10/30 Write block 69 len 16384 data: 7 7 7 6
142212024-02-15T03:03:48.728Z11/30 Write block 318 len 32768 data: 10 9 9 9 9 10 8 10
142222024-02-15T03:03:48.728Z12/30 Write block 567 len 16384 data: 5 5 5 8
142232024-02-15T03:03:48.729Z13/30 Flush
142242024-02-15T03:03:48.729Z14/30 Write block 351 len 20480 data: 13 13 12 11 11
142252024-02-15T03:03:48.729Z15/30 Read block 258 len 12288
142262024-02-15T03:03:48.729Z16/30 Write block 363 len 12288 data: 8 10 10
142272024-02-15T03:03:48.739Z17/30 Write block 316 len 8192 data: 10 11
142282024-02-15T03:03:48.739Z18/30 Read block 481 len 32768
142292024-02-15T03:03:48.739Z19/30 Flush
142302024-02-15T03:03:48.739Z20/30 Write block 514 len 4096 data: 11
142312024-02-15T03:03:48.739Z21/30 Read block 35 len 40960
142322024-02-15T03:03:48.739Z22/30 Read block 317 len 36864
142332024-02-15T03:03:48.739Z23/30 Read block 554 len 28672
142342024-02-15T03:03:48.739Z24/30 Write block 107 len 24576 data: 8 7 6 6 7 6
142352024-02-15T03:03:48.739Z25/30 Read block 78 len 12288
142362024-02-15T03:03:48.739Z26/30 Read block 235 len 24576
142372024-02-15T03:03:48.739Z27/30 Write block 184 len 32768 data: 9 11 12 9 9 12 11 10
142382024-02-15T03:03:48.739Z28/30 Read block 283 len 28672
142392024-02-15T03:03:48.739Z29/30 Write block 129 len 28672 data: 8 8 9 11 12 11 11
142402024-02-15T03:03:48.739Z30/30 Read block 393 len 12288
142412024-02-15T03:03:48.739Z----------------------------------------------------------------
142422024-02-15T03:03:48.739Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
142432024-02-15T03:03:48.739ZStates: Active Active Active
142442024-02-15T03:03:48.739ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142452024-02-15T03:03:48.739Z 1 Acked 1000 Write 2 Done Sent Done false
142462024-02-15T03:03:48.739Z 2 Acked 1001 Write 5 Done Sent Done false
142472024-02-15T03:03:48.740Z 3 Acked 1002 Read 2 Done Sent Done false
142482024-02-15T03:03:48.740Z 4 Acked 1003 Read 4 Done Sent Done false
142492024-02-15T03:03:48.740Z 5 Acked 1004 Write 4 Done Sent Done false
142502024-02-15T03:03:48.740Z 6 Acked 1005 Flush 0 Done Sent Done false
142512024-02-15T03:03:48.740Z 7 Acked 1006 Write 1 Done Sent Done false
142522024-02-15T03:03:48.740Z 8 Acked 1007 Read 3 Done Sent Done false
142532024-02-15T03:03:48.740Z 9 Acked 1008 Read 10 Done Sent Done false
142542024-02-15T03:03:48.740Z 10 Acked 1009 Write 4 Done Sent Done false
142552024-02-15T03:03:48.740Z 11 Acked 1010 Write 8 Done Sent Done false
142562024-02-15T03:03:48.740Z 12 Acked 1011 Write 4 Done Sent Done false
142572024-02-15T03:03:48.740Z 13 Acked 1012 Flush 0 Done Sent Done false
142582024-02-15T03:03:48.740Z 14 Acked 1013 Write 5 Done Sent Done false
142592024-02-15T03:03:48.740Z 15 Acked 1014 Read 3 Done Sent Done false
142602024-02-15T03:03:48.740Z 16 Acked 1015 Write 3 Done Sent Done false
142612024-02-15T03:03:48.740Z 17 Acked 1016 Write 2 Done Sent Done false
142622024-02-15T03:03:48.740Z 18 Acked 1017 Read 8 Done Sent Done false
142632024-02-15T03:03:48.740Z 19 Acked 1018 Flush 0 Done Sent Done false
142642024-02-15T03:03:48.740Z 20 Acked 1019 Write 1 Done Sent Done false
142652024-02-15T03:03:48.740Z 21 Acked 1020 Read 10 Done Sent Done false
142662024-02-15T03:03:48.740Z 22 Acked 1021 Read 9 Done Sent Done false
142672024-02-15T03:03:48.740Z 23 Acked 1022 Read 7 Done Sent Done false
142682024-02-15T03:03:48.740Z 24 Acked 1023 Write 6 Done Sent Done false
142692024-02-15T03:03:48.740Z 25 Acked 1024 Read 3 Done Sent Done false
142702024-02-15T03:03:48.740Z 26 Acked 1025 Read 6 Done Sent Done false
142712024-02-15T03:03:48.741Z 27 Acked 1026 Write 8 Done Sent Done false
142722024-02-15T03:03:48.741Z 28 Acked 1027 Read 7 Done Sent Done false
142732024-02-15T03:03:48.741Z 29 Acked 1028 Write 7 Done Sent Done false
142742024-02-15T03:03:48.741Z 30 Acked 1029 Read 3 Done Sent Sent false
142752024-02-15T03:03:48.741Z STATES DS:0 DS:1 DS:2 TOTAL
142762024-02-15T03:03:48.741Z New 0 0 0 0
142772024-02-15T03:03:48.741Z Sent 0 30 1 31
142782024-02-15T03:03:48.741Z Done 30 0 29 59
142792024-02-15T03:03:48.741Z Skipped 0 0 0 0
142802024-02-15T03:03:48.741Z Error 0 0 0 0
142812024-02-15T03:03:48.741ZLast Flush: 1018 0 1018
142822024-02-15T03:03:48.741ZDownstairs last five completed:
142832024-02-15T03:03:48.741ZUpstairs last five completed: 30 29 28 27 26
142842024-02-15T03:03:48.741Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:47.225402637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1650}
142852024-02-15T03:03:48.741ZWrote out file "/tmp/test_repair_verify.data"
142862024-02-15T03:03:48.742Z
142872024-02-15T03:03:48.742ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142882024-02-15T03:03:48.742ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142892024-02-15T03:03:48.742Z 0 000-019 124 124 124 205 205 205 F F F
142902024-02-15T03:03:48.742Z 1 020-039 126 126 126 208 208 208 F F F
142912024-02-15T03:03:48.742Z 2 040-059 126 126 126 208 208 208 F F F
142922024-02-15T03:03:48.742Z 3 060-079 128 120 128 212 201 212 F F F <---
142932024-02-15T03:03:48.742Z 4 080-099 127 127 127 210 210 210 F F F
142942024-02-15T03:03:48.742Z 5 100-119 127 127 127 210 210 210 T F T <---
142952024-02-15T03:03:48.742Z 6 120-139 125 125 125 206 206 206 T F T <---
142962024-02-15T03:03:48.742Z 7 140-159 125 125 125 206 206 206 F F F
142972024-02-15T03:03:48.742Z 8 160-179 126 126 126 208 208 208 F F F
142982024-02-15T03:03:48.742Z 9 180-199 126 126 126 208 208 208 T F T <---
142992024-02-15T03:03:48.742Z 10 200-219 122 122 122 204 204 204 F F F
143002024-02-15T03:03:48.742Z 11 220-239 127 127 127 210 210 210 F F F
143012024-02-15T03:03:48.742Z 12 240-259 127 127 127 210 210 210 F F F
143022024-02-15T03:03:48.742Z 13 260-279 127 127 127 210 210 210 F F F
143032024-02-15T03:03:48.742Z 14 280-299 122 122 122 204 204 204 F F F
143042024-02-15T03:03:48.742Z 15 300-319 128 121 128 213 203 213 F F F <---
143052024-02-15T03:03:48.742Z 16 320-339 128 126 128 212 208 212 F F F <---
143062024-02-15T03:03:48.742Z 17 340-359 128 127 128 213 210 213 F F F <---
143072024-02-15T03:03:48.742Z 18 360-379 128 126 128 213 209 213 F F F <---
143082024-02-15T03:03:48.742Z 19 380-399 125 125 125 206 206 206 F F F
143092024-02-15T03:03:48.743Z 20 400-419 124 124 124 205 205 205 F F F
143102024-02-15T03:03:48.743Z 21 420-439 124 124 124 205 205 205 F F F
143112024-02-15T03:03:48.743Z 22 440-459 125 125 125 206 206 206 F F F
143122024-02-15T03:03:48.743Z 23 460-479 126 126 126 207 207 207 F F F
143132024-02-15T03:03:48.743Z 24 480-499 125 125 125 206 206 206 F F F
143142024-02-15T03:03:48.743Z 25 500-519 124 124 124 205 205 205 T F T <---
143152024-02-15T03:03:48.743Z 26 520-539 122 122 122 204 204 204 F F F
143162024-02-15T03:03:48.743Z 27 540-559 119 119 119 200 200 200 F F F
143172024-02-15T03:03:48.743Z 28 560-579 128 122 128 212 204 212 F F F <---
143182024-02-15T03:03:48.743Z 29 580-599 128 122 128 212 204 212 F F F <---
143192024-02-15T03:03:48.743ZMax gen: 128, Max flush: 213
143202024-02-15T03:03:48.743ZError: Difference in extent metadata found!
143212024-02-15T03:03:48.743ZA Difference in extent metadata is expected here
143222024-02-15T03:03:48.743ZOn loop 64
143232024-02-15T03:03:48.743Z
143242024-02-15T03:03:48.743ZVerifying data now
143252024-02-15T03:03:54.441ZNo change after verify
143262024-02-15T03:03:54.441ZLoop: 64 Downstairs dump after verify (and repair):
143272024-02-15T03:03:54.525ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143282024-02-15T03:03:54.525Z 0 000-019 124 124 124 205 205 205 F F F
143292024-02-15T03:03:54.525Z 1 020-039 126 126 126 208 208 208 F F F
143302024-02-15T03:03:54.526Z 2 040-059 126 126 126 208 208 208 F F F
143312024-02-15T03:03:54.526Z 3 060-079 128 128 128 212 212 212 F F F
143322024-02-15T03:03:54.526Z 4 080-099 127 127 127 210 210 210 F F F
143332024-02-15T03:03:54.526Z 5 100-119 129 129 129 214 214 214 F F F
143342024-02-15T03:03:54.527Z 6 120-139 129 129 129 214 214 214 F F F
143352024-02-15T03:03:54.527Z 7 140-159 125 125 125 206 206 206 F F F
143362024-02-15T03:03:54.527Z 8 160-179 126 126 126 208 208 208 F F F
143372024-02-15T03:03:54.527Z 9 180-199 129 129 129 214 214 214 F F F
143382024-02-15T03:03:54.527Z 10 200-219 122 122 122 204 204 204 F F F
143392024-02-15T03:03:54.527Z 11 220-239 127 127 127 210 210 210 F F F
143402024-02-15T03:03:54.528Z 12 240-259 127 127 127 210 210 210 F F F
143412024-02-15T03:03:54.528Z 13 260-279 127 127 127 210 210 210 F F F
143422024-02-15T03:03:54.528Z 14 280-299 122 122 122 204 204 204 F F F
143432024-02-15T03:03:54.529Z 15 300-319 128 128 128 213 213 213 F F F
143442024-02-15T03:03:54.529Z 16 320-339 128 128 128 212 212 212 F F F
143452024-02-15T03:03:54.529Z 17 340-359 128 128 128 213 213 213 F F F
143462024-02-15T03:03:54.529Z 18 360-379 128 128 128 213 213 213 F F F
143472024-02-15T03:03:54.529Z 19 380-399 125 125 125 206 206 206 F F F
143482024-02-15T03:03:54.529Z 20 400-419 124 124 124 205 205 205 F F F
143492024-02-15T03:03:54.530Z 21 420-439 124 124 124 205 205 205 F F F
143502024-02-15T03:03:54.530Z 22 440-459 125 125 125 206 206 206 F F F
143512024-02-15T03:03:54.530Z 23 460-479 126 126 126 207 207 207 F F F
143522024-02-15T03:03:54.530Z 24 480-499 125 125 125 206 206 206 F F F
143532024-02-15T03:03:54.530Z 25 500-519 129 129 129 214 214 214 F F F
143542024-02-15T03:03:54.530Z 26 520-539 122 122 122 204 204 204 F F F
143552024-02-15T03:03:54.531Z 27 540-559 119 119 119 200 200 200 F F F
143562024-02-15T03:03:54.531Z 28 560-579 128 128 128 212 212 212 F F F
143572024-02-15T03:03:54.531Z 29 580-599 128 128 128 212 212 212 F F F
143582024-02-15T03:03:54.532ZMax gen: 129, Max flush: 214
143592024-02-15T03:03:54.532Z
143602024-02-15T03:03:54.532ZBegin loop 65
143612024-02-15T03:03:54.532ZDownstairs to restart: 1
143622024-02-15T03:03:54.532Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869043233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658}
143632024-02-15T03:03:54.532Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869334336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658}
143642024-02-15T03:03:54.532Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869365312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658}
143652024-02-15T03:03:54.533Z{"msg":"Crucible 9b3e6b97-a36d-4bb6-b181-7dad419dce62 has session id: 4a94be6d-8899-458e-aef4-45ddf977a9d3","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.86938272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143662024-02-15T03:03:54.533Z{"msg":"Upstairs opts: Upstairs UUID: 9b3e6b97-a36d-4bb6-b181-7dad419dce62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869424775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143672024-02-15T03:03:54.533Z{"msg":"Crucible stats registered with UUID: 9b3e6b97-a36d-4bb6-b181-7dad419dce62","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869453171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143682024-02-15T03:03:54.533ZCrucible runtime is spawned
143692024-02-15T03:03:54.533Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869632519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658}
143702024-02-15T03:03:54.534Z{"msg":"9b3e6b97-a36d-4bb6-b181-7dad419dce62 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869727077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143712024-02-15T03:03:54.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869758953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143722024-02-15T03:03:54.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869791299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143732024-02-15T03:03:54.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869822155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143742024-02-15T03:03:54.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869855481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143752024-02-15T03:03:54.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869873299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143762024-02-15T03:03:54.535Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869893006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143772024-02-15T03:03:54.535Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869921983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143782024-02-15T03:03:54.535Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.869980165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143792024-02-15T03:03:54.535Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.87001796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143802024-02-15T03:03:54.535Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:51.870075443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143812024-02-15T03:03:54.535Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:51.870135756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143822024-02-15T03:03:54.535Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:51.87018175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143832024-02-15T03:03:54.535Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870197488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143842024-02-15T03:03:54.535Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870211846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143852024-02-15T03:03:54.535Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870247352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143862024-02-15T03:03:54.536Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870267039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143872024-02-15T03:03:54.536Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870307194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143882024-02-15T03:03:54.536Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.8708217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143892024-02-15T03:03:54.536Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870862365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143902024-02-15T03:03:54.536Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870890221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143912024-02-15T03:03:54.536Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.870924217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143922024-02-15T03:03:54.536Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.871405787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143932024-02-15T03:03:54.536Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.871511843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143942024-02-15T03:03:54.536Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.871951558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143952024-02-15T03:03:54.536Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.871988344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143962024-02-15T03:03:54.536Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.872003962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143972024-02-15T03:03:54.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.872069043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143982024-02-15T03:03:54.537Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.87209626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
143992024-02-15T03:03:54.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:51.872112738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144002024-02-15T03:03:54.537Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.871865582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144012024-02-15T03:03:54.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.871978128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144022024-02-15T03:03:54.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.872882565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144032024-02-15T03:03:54.537Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.872940948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144042024-02-15T03:03:54.537Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.873449684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144052024-02-15T03:03:54.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874093123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144062024-02-15T03:03:54.537Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874154915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144072024-02-15T03:03:54.537Z{"msg":"[0]R flush_numbers[0..12]: [205, 208, 208, 212, 210, 214, 214, 206, 208, 214, 204, 210]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874192291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144082024-02-15T03:03:54.537Z{"msg":"[0]R generation[0..12]: [124, 126, 126, 128, 127, 129, 129, 125, 126, 129, 122, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874225376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144092024-02-15T03:03:54.538Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874266631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144102024-02-15T03:03:54.538Z{"msg":"[1]R flush_numbers[0..12]: [205, 208, 208, 212, 210, 214, 214, 206, 208, 214, 204, 210]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874309156ZWait for a query_work_queue command to finish before sending IO
144112024-02-15T03:03:54.538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144122024-02-15T03:03:54.538Z{"msg":"[1]R generation[0..12]: [124, 126, 126, 128, 127, 129, 129, 125, 126, 129, 122, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.87435503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144132024-02-15T03:03:54.538Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874394605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144142024-02-15T03:03:54.538Z{"msg":"[2]R flush_numbers[0..12]: [205, 208, 208, 212, 210, 214, 214, 206, 208, 214, 204, 210]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.87443527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144152024-02-15T03:03:54.538Z{"msg":"[2]R generation[0..12]: [124, 126, 126, 128, 127, 129, 129, 125, 126, 129, 122, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874474455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144162024-02-15T03:03:54.538Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.87451424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144172024-02-15T03:03:54.538Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874552455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144182024-02-15T03:03:54.538Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874592081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144192024-02-15T03:03:54.538Z{"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874630196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144202024-02-15T03:03:54.538Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874667651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144212024-02-15T03:03:54.539Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874706596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144222024-02-15T03:03:54.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874744142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144232024-02-15T03:03:54.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144242024-02-15T03:03:54.539Ztime":"2024-02-15T03:03:52.874783326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144252024-02-15T03:03:54.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874826841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144262024-02-15T03:03:54.539Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874866056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144272024-02-15T03:03:54.969Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874886414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144282024-02-15T03:03:54.969Z{"msg":"9b3e6b97-a36d-4bb6-b181-7dad419dce62 is now active with session: 4a94be6d-8899-458e-aef4-45ddf977a9d3","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874921639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144292024-02-15T03:03:54.969Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.874957395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658,"session_id":"4a94be6d-8899-458e-aef4-45ddf977a9d3"}
144302024-02-15T03:03:54.970Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:52.875013618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658}
144312024-02-15T03:03:54.970ZLoading write count information from file "/tmp/test_repair_verify.data"
144322024-02-15T03:03:54.970ZRun Repair workload
144332024-02-15T03:03:54.970Z01/30 Flush
144342024-02-15T03:03:54.970Z02/30 Write block 287 len 24576 data: 13 9 11 18 16 15
144352024-02-15T03:03:54.970Z03/30 Write block 346 len 24576 data: 11 10 9 11 12 14
144362024-02-15T03:03:54.970Z04/30 Read block 511 len 36864
144372024-02-15T03:03:54.971Z05/30 Write block 1 len 24576 data: 2 2 6 8 8 9
144382024-02-15T03:03:54.971Z06/30 Write block 396 len 40960 data: 10 9 8 10 8 7 6 5 5 10
144392024-02-15T03:03:54.971Z07/30 Read block 118 len 4096
144402024-02-15T03:03:54.971Z08/30 Read block 557 len 24576
144412024-02-15T03:03:54.971Z09/30 Write block 71 len 8192 data: 8 7
144422024-02-15T03:03:54.971Z10/30 Read block 188 len 20480
144432024-02-15T03:03:54.971Z11/30 Write block 469 len 16384 data: 12 14 12 10
144442024-02-15T03:03:54.972Z12/30 Write block 589 len 36864 data: 13 14 16 13 13 13 11 8 6
144452024-02-15T03:03:54.972Z13/30 Write block 41 len 20480 data: 9 10 9 11 11
144462024-02-15T03:03:54.972Z14/30 Write block 171 len 12288 data: 8 6 5
144472024-02-15T03:03:54.972Z15/30 Write block 242 len 4096 data: 13
144482024-02-15T03:03:54.972Z16/30 Read block 149 len 32768
144492024-02-15T03:03:54.972Z17/30 Read block 354 len 8192
144502024-02-15T03:03:54.972Z18/30 Read block 172 len 12288
144512024-02-15T03:03:54.973Z19/30 Read block 291 len 32768
144522024-02-15T03:03:54.973Z20/30 Write block 219 len 8192 data: 12 13
144532024-02-15T03:03:54.973Z21/30 Read block 565 len 4096
144542024-02-15T03:03:54.973Z22/30 Read block 61 len 8192
144552024-02-15T03:03:54.973Z23/30 Read block 182 len 12288
144562024-02-15T03:03:54.973Z24/30 Read block 338 len 24576
144572024-02-15T03:03:54.974Z25/30 Read block 566 len 32768
144582024-02-15T03:03:54.974Z26/30 Write block 32 len 16384 data: 8 10 10 12
144592024-02-15T03:03:54.974Z27/30 Read block 422 len 28672
144602024-02-15T03:03:54.974Z28/30 Write block 57 len 16384 data: 12 13 10 10
144612024-02-15T03:03:54.974Z29/30 Read block 452 len 4096
144622024-02-15T03:03:54.975Z30/30 Read block 82 len 32768
144632024-02-15T03:03:54.975Z----------------------------------------------------------------
144642024-02-15T03:03:54.975Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:29
144652024-02-15T03:03:54.975ZStates: Active Active Active
144662024-02-15T03:03:54.975ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144672024-02-15T03:03:54.975Z 2 Acked 1001 Write 6 Done Done Done false
144682024-02-15T03:03:54.975Z 3 Acked 1002 Write 6 Done Done Done false
144692024-02-15T03:03:54.976Z 4 Acked 1003 Read 9 Done Sent Done false
144702024-02-15T03:03:54.976Z 5 Acked 1004 Write 6 Done Sent Done false
144712024-02-15T03:03:54.976Z 6 Acked 1005 Write 10 Done Sent Done false
144722024-02-15T03:03:54.976Z 7 Acked 1006 Read 1 Done Sent Done false
144732024-02-15T03:03:54.976Z 8 Acked 1007 Read 6 Done Sent Done false
144742024-02-15T03:03:54.977Z 9 Acked 1008 Write 2 Done Sent Done false
144752024-02-15T03:03:54.977Z 10 Acked 1009 Read 5 Done Sent Done false
144762024-02-15T03:03:54.977Z 11 Acked 1010 Write 4 Done Sent Done false
144772024-02-15T03:03:54.977Z 12 Acked 1011 Write 9 Done Sent Done false
144782024-02-15T03:03:54.977Z 13 Acked 1012 Write 5 Done Sent Done false
144792024-02-15T03:03:54.977Z 14 Acked 1013 Write 3 Done Sent Done false
144802024-02-15T03:03:54.977Z 15 Acked 1014 Write 1 Done Sent Done false
144812024-02-15T03:03:54.978Z 16 Acked 1015 Read 8 Done Sent Done false
144822024-02-15T03:03:54.978Z 17 Acked 1016 Read 2 Done Sent Done false
144832024-02-15T03:03:54.978Z 18 Acked 1017 Read 3 Done Sent Done false
144842024-02-15T03:03:54.978Z 19 Acked 1018 Read 8 Done Sent Done false
144852024-02-15T03:03:54.978Z 20 Acked 1019 Write 2 Done Sent Done false
144862024-02-15T03:03:54.978Z 21 Acked 1020 Read 1 Done Sent Done false
144872024-02-15T03:03:54.978Z 22 Acked 1021 Read 2 Done Sent Done false
144882024-02-15T03:03:54.979Z 23 Acked 1022 Read 3 Done Sent Done false
144892024-02-15T03:03:54.979Z 24 Acked 1023 Read 6 Done Sent Done false
144902024-02-15T03:03:54.979Z 25 Acked 1024 Read 8 Done Sent Done false
144912024-02-15T03:03:54.979Z 26 Acked 1025 Write 4 Done Sent Done false
144922024-02-15T03:03:54.979Z 27 Acked 1026 Read 7 Done Sent Done false
144932024-02-15T03:03:54.979Z 28 Acked 1027 Write 4 Done Sent Done false
144942024-02-15T03:03:54.980Z 29 Acked 1028 Read 1 Done Sent Done false
144952024-02-15T03:03:54.980Z 30 Acked 1029 Read 8 Done Sent Done false
144962024-02-15T03:03:54.980Z STATES DS:0 DS:1 DS:2 TOTAL
144972024-02-15T03:03:54.980Z New 0 0 0 0
144982024-02-15T03:03:54.980Z Sent 0 27 0 27
144992024-02-15T03:03:54.980Z Done 29 2 29 60
145002024-02-15T03:03:54.980Z Skipped 0 0 0 0
145012024-02-15T03:03:54.981Z Error 0 0 0 0
145022024-02-15T03:03:54.981ZLast Flush: 1000 1000 1000
145032024-02-15T03:03:54.981ZDownstairs last five completed: 1000
145042024-02-15T03:03:54.981ZUpstairs last five completed: 30 29 28 27 26
145052024-02-15T03:03:54.981Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:52.900483626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1658}
145062024-02-15T03:03:54.982ZWrote out file "/tmp/test_repair_verify.data"
145072024-02-15T03:03:54.982Z
145082024-02-15T03:03:54.982ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145092024-02-15T03:03:54.982ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145102024-02-15T03:03:54.982Z 0 000-019 124 124 124 205 205 205 T F T <---
145112024-02-15T03:03:54.982Z 1 020-039 126 126 126 208 208 208 T F T <---
145122024-02-15T03:03:54.983Z 2 040-059 126 126 126 208 208 208 T F T <---
145132024-02-15T03:03:54.983Z 3 060-079 128 128 128 212 212 212 T F T <---
145142024-02-15T03:03:54.983Z 4 080-099 127 127 127 210 210 210 F F F
145152024-02-15T03:03:54.983Z 5 100-119 129 129 129 214 214 214 F F F
145162024-02-15T03:03:54.983Z 6 120-139 129 129 129 214 214 214 F F F
145172024-02-15T03:03:54.983Z 7 140-159 125 125 125 206 206 206 F F F
145182024-02-15T03:03:54.983Z 8 160-179 126 126 126 208 208 208 T F T <---
145192024-02-15T03:03:54.983Z 9 180-199 129 129 129 214 214 214 F F F
145202024-02-15T03:03:54.984Z 10 200-219 122 122 122 204 204 204 T F T <---
145212024-02-15T03:03:54.984Z 11 220-239 127 127 127 210 210 210 T F T <---
145222024-02-15T03:03:54.984Z 12 240-259 127 127 127 210 210 210 T F T <---
145232024-02-15T03:03:54.984Z 13 260-279 127 127 127 210 210 210 F F F
145242024-02-15T03:03:54.984Z 14 280-299 122 122 122 204 204 204 T T T
145252024-02-15T03:03:54.984Z 15 300-319 128 128 128 213 213 213 F F F
145262024-02-15T03:03:54.984Z 16 320-339 128 128 128 212 212 212 F F F
145272024-02-15T03:03:55.009Z 17 340-359 128 128 128 213 213 213 T T T
145282024-02-15T03:03:55.010Z 18 360-379 128 128 128 213 213 213 F F F
145292024-02-15T03:03:55.010Z 19 380-399 125 125 125 206 206 206 T F T <---
145302024-02-15T03:03:55.010Z 20 400-419 124 124 124 205 205 205 T F T <---
145312024-02-15T03:03:55.010Z 21 420-439 124 124 124 205 205 205 F F F
145322024-02-15T03:03:55.010Z 22 440-459 125 125 125 206 206 206 F F F
145332024-02-15T03:03:55.010Z 23 460-479 126 126 126 207 207 207 T F T <---
145342024-02-15T03:03:55.010Z 24 480-499 125 125 125 206 206 206 F F F
145352024-02-15T03:03:55.011Z 25 500-519 129 129 129 214 214 214 F F F
145362024-02-15T03:03:55.011Z 26 520-539 122 122 122 204 204 204 F F F
145372024-02-15T03:03:55.011Z 27 540-559 119 119 119 200 200 200 F F F
145382024-02-15T03:03:55.011Z 28 560-579 128 128 128 212 212 212 F F F
145392024-02-15T03:03:55.011Z 29 580-599 128 128 128 212 212 212 T F T <---
145402024-02-15T03:03:55.012ZMax gen: 129, Max flush: 214
145412024-02-15T03:03:55.012ZError: Difference in extent metadata found!
145422024-02-15T03:03:55.012ZA Difference in extent metadata is expected here
145432024-02-15T03:03:55.012ZOn loop 65
145442024-02-15T03:03:55.012Z
145452024-02-15T03:03:55.012ZVerifying data now
145462024-02-15T03:03:59.761ZNo change after verify
145472024-02-15T03:03:59.761ZLoop: 65 Downstairs dump after verify (and repair):
145482024-02-15T03:03:59.786ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145492024-02-15T03:03:59.786Z 0 000-019 130 130 130 215 215 215 F F F
145502024-02-15T03:03:59.786Z 1 020-039 130 130 130 215 215 215 F F F
145512024-02-15T03:03:59.786Z 2 040-059 130 130 130 215 215 215 F F F
145522024-02-15T03:03:59.786Z 3 060-079 130 130 130 215 215 215 F F F
145532024-02-15T03:03:59.786Z 4 080-099 127 127 127 210 210 210 F F F
145542024-02-15T03:03:59.786Z 5 100-119 129 129 129 214 214 214 F F F
145552024-02-15T03:03:59.804Z 6 120-139 129 129 129 214 214 214 F F F
145562024-02-15T03:03:59.804Z 7 140-159 125 125 125 206 206 206 F F F
145572024-02-15T03:03:59.805Z 8 160-179 130 130 130 215 215 215 F F F
145582024-02-15T03:03:59.805Z 9 180-199 129 129 129 214 214 214 F F F
145592024-02-15T03:03:59.805Z 10 200-219 130 130 130 215 215 215 F F F
145602024-02-15T03:03:59.805Z 11 220-239 130 130 130 215 215 215 F F F
145612024-02-15T03:03:59.805Z 12 240-259 130 130 130 215 215 215 F F F
145622024-02-15T03:03:59.806Z 13 260-279 127 127 127 210 210 210 F F F
145632024-02-15T03:03:59.806Z 14 280-299 130 130 130 215 215 215 F F F
145642024-02-15T03:03:59.806Z 15 300-319 128 128 128 213 213 213 F F F
145652024-02-15T03:03:59.806Z 16 320-339 128 128 128 212 212 212 F F F
145662024-02-15T03:03:59.806Z 17 340-359 130 130 130 215 215 215 F F F
145672024-02-15T03:03:59.806Z 18 360-379 128 128 128 213 213 213 F F F
145682024-02-15T03:03:59.806Z 19 380-399 130 130 130 215 215 215 F F F
145692024-02-15T03:03:59.806Z 20 400-419 130 130 130 215 215 215 F F F
145702024-02-15T03:03:59.806Z 21 420-439 124 124 124 205 205 205 F F F
145712024-02-15T03:03:59.806Z 22 440-459 125 125 125 206 206 206 F F F
145722024-02-15T03:03:59.807Z 23 460-479 130 130 130 215 215 215 F F F
145732024-02-15T03:03:59.807Z 24 480-499 125 125 125 206 206 206 F F F
145742024-02-15T03:03:59.807Z 25 500-519 129 129 129 214 214 214 F F F
145752024-02-15T03:03:59.807Z 26 520-539 122 122 122 204 204 204 F F F
145762024-02-15T03:03:59.807Z 27 540-559 119 119 119 200 200 200 F F F
145772024-02-15T03:03:59.807Z 28 560-579 128 128 128 212 212 212 F F F
145782024-02-15T03:03:59.807Z 29 580-599 130 130 130 215 215 215 F F F
145792024-02-15T03:03:59.807ZMax gen: 130, Max flush: 215
145802024-02-15T03:03:59.807Z
145812024-02-15T03:03:59.807ZBegin loop 66
145822024-02-15T03:03:59.807ZDownstairs to restart: 0
145832024-02-15T03:03:59.833Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.698586332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666}
145842024-02-15T03:03:59.833Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.698943237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666}
145852024-02-15T03:03:59.833Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.698966904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666}
145862024-02-15T03:03:59.833ZCrucible runtime is spawned
145872024-02-15T03:03:59.833Z{"msg":"Crucible b0f49fda-0319-48b1-b96c-3b77390eb50c has session id: d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699016418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145882024-02-15T03:03:59.834Z{"msg":"Upstairs opts: Upstairs UUID: b0f49fda-0319-48b1-b96c-3b77390eb50c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.69907451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145892024-02-15T03:03:59.834Z{"msg":"Crucible stats registered with UUID: b0f49fda-0319-48b1-b96c-3b77390eb50c","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699110666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145902024-02-15T03:03:59.834Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699128293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666}
145912024-02-15T03:03:59.834Z{"msg":"b0f49fda-0319-48b1-b96c-3b77390eb50c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699281014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145922024-02-15T03:03:59.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.69931233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145932024-02-15T03:03:59.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699332868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145942024-02-15T03:03:59.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699354015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145952024-02-15T03:03:59.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699392071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145962024-02-15T03:03:59.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699419857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145972024-02-15T03:03:59.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699460742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145982024-02-15T03:03:59.835Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.69947846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
145992024-02-15T03:03:59.835Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699509416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146002024-02-15T03:03:59.835Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699547311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146012024-02-15T03:03:59.835Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:58.699640799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146022024-02-15T03:03:59.835Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:58.699694702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146032024-02-15T03:03:59.835Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699724299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146042024-02-15T03:03:59.835Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:58.699753805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146052024-02-15T03:03:59.835Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699768633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146062024-02-15T03:03:59.835Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699799689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146072024-02-15T03:03:59.835Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699889798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146082024-02-15T03:03:59.835Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.699960799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146092024-02-15T03:03:59.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.700398324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146102024-02-15T03:03:59.836Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.70043446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146112024-02-15T03:03:59.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.700560714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146122024-02-15T03:03:59.836Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.700620047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146132024-02-15T03:03:59.836Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.700936027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146142024-02-15T03:03:59.836Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.70107395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146152024-02-15T03:03:59.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.701456882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146162024-02-15T03:03:59.836Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.701503416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146172024-02-15T03:03:59.836Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.701529953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146182024-02-15T03:03:59.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.701560319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146192024-02-15T03:03:59.836Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.701584536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146202024-02-15T03:03:59.837Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:58.701614532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146212024-02-15T03:04:00.807Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.701594318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146222024-02-15T03:04:00.808Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.701834488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146232024-02-15T03:04:00.835Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.702838762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146242024-02-15T03:04:00.836Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.702894705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146252024-02-15T03:04:00.836Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.703366256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146262024-02-15T03:04:00.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704024193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146272024-02-15T03:04:00.836Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704072187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146282024-02-15T03:04:00.836Z{"msg":"[0]R flush_numbers[0..12]: [215, 215, 215, 215, 210, 214, 214, 206, 215, 214, 215, 215]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704093155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146292024-02-15T03:04:00.836Z{"msg":"[0]R generation[0..12]: [130, 130, 130, 130, 127, 129, 129, 125, 130, 129, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704111222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146302024-02-15T03:04:00.836Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704140289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146312024-02-15T03:04:00.836Z{"msg":"[1]R flush_numbers[0..12]: [215, 215, 215, 215, 210, 214, 214, 206, 215, 214, 215, 215]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704158646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146322024-02-15T03:04:00.836Z{"msg":"[1]R generation[0..12]: [130, 130, 130, 130, 127, 129, 129, 125, 130, 129, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704184473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146332024-02-15T03:04:00.837Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704204161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146342024-02-15T03:04:00.837Z{"msg":"[2]R flush_numbers[0..12]: [215, 215, 215, 215, 210, 214, 214, 206, 215, 214, 215, 215]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
146352024-02-15T03:04:00.837Z,"time":"2024-02-15T03:03:59.704222058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146362024-02-15T03:04:00.837Z{"msg":"[2]R generation[0..12]: [130, 130, 130, 130, 127, 129, 129, 125, 130, 129, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704302398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146372024-02-15T03:04:00.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704341603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146382024-02-15T03:04:00.838Z{"msg":"Max found gen is 131","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704383908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146392024-02-15T03:04:00.838Z{"msg":"Generation requested: 132 >= found:131","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704425423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146402024-02-15T03:04:00.838Z{"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704471057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146412024-02-15T03:04:00.838Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704515542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146422024-02-15T03:04:00.838Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704560456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146432024-02-15T03:04:00.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146442024-02-15T03:04:00.838Z,"time":"2024-02-15T03:03:59.704602171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146452024-02-15T03:04:00.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704642886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146462024-02-15T03:04:00.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704663983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146472024-02-15T03:04:00.839Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704681411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146482024-02-15T03:04:00.839Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704706458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146492024-02-15T03:04:00.839Z{"msg":"b0f49fda-0319-48b1-b96c-3b77390eb50c is now active with session: d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704724626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146502024-02-15T03:04:00.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.704749722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666,"session_id":"d1eb8ae6-9ed3-40f7-8592-f06781b0c3f8"}
146512024-02-15T03:04:00.839Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:03:59.70476585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666}
146522024-02-15T03:04:00.840ZLoading write count information from file "/tmp/test_repair_verify.data"
146532024-02-15T03:04:00.840ZRun Repair workload
146542024-02-15T03:04:00.840Z01/30 Write block 416 len 36864 data: 10 9 9 10 11 10 9 11 9
146552024-02-15T03:04:00.840Z02/30 Write block 31 len 28672 data: 9 9 11 11 13 11 11
146562024-02-15T03:04:00.840Z03/30 Flush
146572024-02-15T03:04:00.840Z04/30 Write block 30 len 28672 data: 9 10 10 12 12 14 12
146582024-02-15T03:04:00.840Z05/30 Flush
146592024-02-15T03:04:00.840Z06/30 Write block 565 len 32768 data: 11 6 6 6 6 9 10 10
146602024-02-15T03:04:00.840Z07/30 Flush
146612024-02-15T03:04:00.850Z08/30 Write block 239 len 20480 data: 11 11 12 14 11
146622024-02-15T03:04:00.850Z09/30 Write block 10 len 28672 data: 13 8 6 5 6 5 5
146632024-02-15T03:04:00.850Z10/30 Write block 330 len 40960 data: 13 14 13 13 16 15 13 13 13 9
146642024-02-15T03:04:00.851Z11/30 Read block 527 len 40960
146652024-02-15T03:04:00.851Z12/30 Read block 494 len 12288
146662024-02-15T03:04:00.851Z13/30 Write block 466 len 20480 data: 11 10 10 13 15
146672024-02-15T03:04:00.851Z14/30 Write block 530 len 24576 data: 12 8 11 9 8 9
146682024-02-15T03:04:00.851Z15/30 Read block 561 len 16384
146692024-02-15T03:04:00.851Z16/30 Flush
146702024-02-15T03:04:00.851Z17/30 Write block 259 len 8192 data: 12 9
146712024-02-15T03:04:00.851Z18/30 Read block 90 len 24576
146722024-02-15T03:04:00.852Z19/30 Read block 346 len 16384
146732024-02-15T03:04:00.852Z20/30 Read block 274 len 4096
146742024-02-15T03:04:00.852Z21/30 Write block 362 len 4096 data: 11
146752024-02-15T03:04:00.852Z22/30 Write block 480 len 8192 data: 12 12
146762024-02-15T03:04:00.852Z23/30 Write block 43 len 20480 data: 10 12 12 10 12
146772024-02-15T03:04:00.852Z24/30 Read block 575 len 20480
146782024-02-15T03:04:00.852Z25/30 Read block 218 len 36864
146792024-02-15T03:04:00.852Z26/30 Write block 122 len 8192 data: 7 8
146802024-02-15T03:04:00.852Z27/30 Write block 515 len 20480 data: 8 9 8 7 6
146812024-02-15T03:04:00.852Z28/30 Write block 265 len 36864 data: 8 8 7 8 9 11 10 12 11
146822024-02-15T03:04:00.853Z29/30 Write block 486 len 28672 data: 7 9 7 10 12 11 13
146832024-02-15T03:04:00.853Z30/30 Read block 179 len 8192
146842024-02-15T03:04:00.853Z----------------------------------------------------------------
146852024-02-15T03:04:00.853Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
146862024-02-15T03:04:00.853ZStates: Active Active Active
146872024-02-15T03:04:00.853ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
146882024-02-15T03:04:00.853Z 1 Acked 1000 Write 9 Sent Done Done false
146892024-02-15T03:04:00.853Z 2 Acked 1001 Write 7 Sent Done Done false
146902024-02-15T03:04:00.853Z 3 Acked 1002 Flush 0 Sent Done Done false
146912024-02-15T03:04:00.854Z 4 Acked 1003 Write 7 Sent Done Done false
146922024-02-15T03:04:00.854Z 5 Acked 1004 Flush 0 Sent Done Done false
146932024-02-15T03:04:00.854Z 6 Acked 1005 Write 8 Sent Done Done false
146942024-02-15T03:04:00.854Z 7 Acked 1006 Flush 0 Sent Done Done false
146952024-02-15T03:04:00.854Z 8 Acked 1007 Write 5 Sent Done Done false
146962024-02-15T03:04:00.854Z 9 Acked 1008 Write 7 Sent Done Done false
146972024-02-15T03:04:00.854Z 10 Acked 1009 Write 10 Sent Done Done false
146982024-02-15T03:04:00.854Z 11 Acked 1010 Read 10 Sent Done Done false
146992024-02-15T03:04:00.854Z 12 Acked 1011 Read 3 Sent Done Done false
147002024-02-15T03:04:00.855Z 13 Acked 1012 Write 5 Sent Done Done false
147012024-02-15T03:04:00.855Z 14 Acked 1013 Write 6 Sent Done Done false
147022024-02-15T03:04:00.855Z 15 Acked 1014 Read 4 Sent Done Done false
147032024-02-15T03:04:00.855Z 16 Acked 1015 Flush 0 Sent Done Done false
147042024-02-15T03:04:00.855Z 17 Acked 1016 Write 2 Sent Done Done false
147052024-02-15T03:04:00.855Z 18 Acked 1017 Read 6 Sent Done Done false
147062024-02-15T03:04:00.855Z 19 Acked 1018 Read 4 Sent Done Done false
147072024-02-15T03:04:00.855Z 20 Acked 1019 Read 1 Sent Done Done false
147082024-02-15T03:04:00.855Z 21 Acked 1020 Write 1 Sent Done Done false
147092024-02-15T03:04:00.855Z 22 Acked 1021 Write 2 Sent Done Done false
147102024-02-15T03:04:00.856Z 23 Acked 1022 Write 5 Sent Done Done false
147112024-02-15T03:04:00.856Z 24 Acked 1023 Read 5 Sent Done Done false
147122024-02-15T03:04:00.856Z 25 Acked 1024 Read 9 Sent Done Done false
147132024-02-15T03:04:00.856Z 26 Acked 1025 Write 2 Sent Done Done false
147142024-02-15T03:04:00.856Z 27 Acked 1026 Write 5 Sent Done Done false
147152024-02-15T03:04:00.856Z 28 Acked 1027 Write 9 Sent Done Done false
147162024-02-15T03:04:00.856Z 29 Acked 1028 Write 7 Sent Done Sent false
147172024-02-15T03:04:00.856Z 30 Acked 1029 Read 2 Sent Done Sent false
147182024-02-15T03:04:00.856Z STATES DS:0 DS:1 DS:2 TOTAL
147192024-02-15T03:04:00.856Z New 0 0 0 0
147202024-02-15T03:04:00.857Z Sent 30 0 2 32
147212024-02-15T03:04:00.857Z Done 0 30 28 58
147222024-02-15T03:04:00.857Z Skipped 0 0 0 0
147232024-02-15T03:04:00.857Z Error 0 0 0 0
147242024-02-15T03:04:00.857ZLast Flush: 0 1015 1015
147252024-02-15T03:04:00.857ZDownstairs last five completed:
147262024-02-15T03:04:00.857ZUpstairs last five completed: 30 29 28 27 26
147272024-02-15T03:04:00.857Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:03:59.740143298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1666}
147282024-02-15T03:04:00.858ZWrote out file "/tmp/test_repair_verify.data"
147292024-02-15T03:04:00.858Z
147302024-02-15T03:04:00.858ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147312024-02-15T03:04:00.880ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147322024-02-15T03:04:00.880Z 0 000-019 130 132 132 215 219 219 F F F <---
147332024-02-15T03:04:00.880Z 1 020-039 130 132 132 215 217 217 F F F <---
147342024-02-15T03:04:00.880Z 2 040-059 130 130 130 215 215 215 F T T <---
147352024-02-15T03:04:00.880Z 3 060-079 130 130 130 215 215 215 F F F
147362024-02-15T03:04:00.880Z 4 080-099 127 127 127 210 210 210 F F F
147372024-02-15T03:04:00.880Z 5 100-119 129 129 129 214 214 214 F F F
147382024-02-15T03:04:00.880Z 6 120-139 129 129 129 214 214 214 F T T <---
147392024-02-15T03:04:00.881Z 7 140-159 125 125 125 206 206 206 F F F
147402024-02-15T03:04:00.881Z 8 160-179 130 130 130 215 215 215 F F F
147412024-02-15T03:04:00.881Z 9 180-199 129 129 129 214 214 214 F F F
147422024-02-15T03:04:00.881Z 10 200-219 130 130 130 215 215 215 F F F
147432024-02-15T03:04:00.881Z 11 220-239 130 132 132 215 219 219 F F F <---
147442024-02-15T03:04:00.881Z 12 240-259 130 132 132 215 219 219 F T T <---
147452024-02-15T03:04:00.885Z 13 260-279 127 127 127 210 210 210 F T T <---
147462024-02-15T03:04:00.886Z 14 280-299 130 130 130 215 215 215 F F F
147472024-02-15T03:04:00.886Z 15 300-319 128 128 128 213 213 213 F F F
147482024-02-15T03:04:00.886Z 16 320-339 128 132 132 212 219 219 F F F <---
147492024-02-15T03:04:00.886Z 17 340-359 130 130 130 215 215 215 F F F
147502024-02-15T03:04:00.886Z 18 360-379 128 128 128 213 213 213 F T T <---
147512024-02-15T03:04:00.886Z 19 380-399 130 130 130 215 215 215 F F F
147522024-02-15T03:04:00.886Z 20 400-419 130 132 132 215 216 216 F F F <---
147532024-02-15T03:04:00.886Z 21 420-439 124 132 132 205 216 216 F F F <---
147542024-02-15T03:04:00.886Z 22 440-459 125 125 125 206 206 206 F F F
147552024-02-15T03:04:00.886Z 23 460-479 130 132 132 215 219 219 F F F <---
147562024-02-15T03:04:00.887Z 24 480-499 125 125 125 206 206 206 F T T <---
147572024-02-15T03:04:00.887Z 25 500-519 129 129 129 214 214 214 F T T <---
147582024-02-15T03:04:00.887Z 26 520-539 122 132 132 204 219 219 F F F <---
147592024-02-15T03:04:00.887Z 27 540-559 119 119 119 200 200 200 F F F
147602024-02-15T03:04:00.887Z 28 560-579 128 132 132 212 218 218 F F F <---
147612024-02-15T03:04:00.887Z 29 580-599 130 130 130 215 215 215 F F F
147622024-02-15T03:04:00.887ZMax gen: 132, Max flush: 219
147632024-02-15T03:04:00.887ZError: Difference in extent metadata found!
147642024-02-15T03:04:00.887ZA Difference in extent metadata is expected here
147652024-02-15T03:04:00.887ZOn loop 66
147662024-02-15T03:04:00.887Z
147672024-02-15T03:04:00.888ZVerifying data now
147682024-02-15T03:04:05.666ZNo change after verify
147692024-02-15T03:04:05.666ZLoop: 66 Downstairs dump after verify (and repair):
147702024-02-15T03:04:05.684ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147712024-02-15T03:04:05.684Z 0 000-019 132 132 132 219 219 219 F F F
147722024-02-15T03:04:05.684Z 1 020-039 132 132 132 217 217 217 F F F
147732024-02-15T03:04:05.684Z 2 040-059 133 133 133 220 220 220 F F F
147742024-02-15T03:04:05.684Z 3 060-079 130 130 130 215 215 215 F F F
147752024-02-15T03:04:05.684Z 4 080-099 127 127 127 210 210 210 F F F
147762024-02-15T03:04:05.684Z 5 100-119 129 129 129 214 214 214 F F F
147772024-02-15T03:04:05.684Z 6 120-139 133 133 133 220 220 220 F F F
147782024-02-15T03:04:05.684Z 7 140-159 125 125 125 206 206 206 F F F
147792024-02-15T03:04:05.684Z 8 160-179 130 130 130 215 215 215 F F F
147802024-02-15T03:04:05.684Z 9 180-199 129 129 129 214 214 214 F F F
147812024-02-15T03:04:05.684Z 10 200-219 130 130 130 215 215 215 F F F
147822024-02-15T03:04:05.688Z 11 220-239 132 132 132 219 219 219 F F F
147832024-02-15T03:04:05.688Z 12 240-259 133 133 133 220 220 220 F F F
147842024-02-15T03:04:05.688Z 13 260-279 133 133 133 220 220 220 F F F
147852024-02-15T03:04:05.688Z 14 280-299 130 130 130 215 215 215 F F F
147862024-02-15T03:04:05.688Z 15 300-319 128 128 128 213 213 213 F F F
147872024-02-15T03:04:05.688Z 16 320-339 132 132 132 219 219 219 F F F
147882024-02-15T03:04:05.688Z 17 340-359 130 130 130 215 215 215 F F F
147892024-02-15T03:04:05.688Z 18 360-379 133 133 133 220 220 220 F F F
147902024-02-15T03:04:05.688Z 19 380-399 130 130 130 215 215 215 F F F
147912024-02-15T03:04:05.688Z 20 400-419 132 132 132 216 216 216 F F F
147922024-02-15T03:04:05.688Z 21 420-439 132 132 132 216 216 216 F F F
147932024-02-15T03:04:05.688Z 22 440-459 125 125 125 206 206 206 F F F
147942024-02-15T03:04:05.688Z 23 460-479 132 132 132 219 219 219 F F F
147952024-02-15T03:04:05.688Z 24 480-499 133 133 133 220 220 220 F F F
147962024-02-15T03:04:05.688Z 25 500-519 133 133 133 220 220 220 F F F
147972024-02-15T03:04:05.688Z 26 520-539 132 132 132 219 219 219 F F F
147982024-02-15T03:04:05.688Z 27 540-559 119 119 119 200 200 200 F F F
147992024-02-15T03:04:05.688Z 28 560-579 132 132 132 218 218 218 F F F
148002024-02-15T03:04:05.689Z 29 580-599 130 130 130 215 215 215 F F F
148012024-02-15T03:04:05.689ZMax gen: 133, Max flush: 220
148022024-02-15T03:04:05.689Z
148032024-02-15T03:04:05.689ZBegin loop 67
148042024-02-15T03:04:05.689ZDownstairs to restart: 0
148052024-02-15T03:04:05.707Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
148062024-02-15T03:04:05.707Ztime":"2024-02-15T03:04:04.607182722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674}
148072024-02-15T03:04:05.738Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607617917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674}
148082024-02-15T03:04:05.738Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607659262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674}
148092024-02-15T03:04:05.738Z{"msg":"Crucible ae624513-b32a-4ddd-8afc-443e3dfeceeb has session id: 390cc217-c828-41ad-87d5-ec9a3cc58897","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607681469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148102024-02-15T03:04:05.738Z{"msg":"Upstairs opts: Upstairs UUID: ae624513-b32a-4ddd-8afc-443e3dfeceeb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607722524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148112024-02-15T03:04:05.738Z{"msg":"Crucible stats registered with UUID: ae624513-b32a-4ddd-8afc-443e3dfeceeb","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607738762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148122024-02-15T03:04:05.738Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.60775792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674}
148132024-02-15T03:04:05.738Z{"msg":"ae624513-b32a-4ddd-8afc-443e3dfeceeb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607776468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148142024-02-15T03:04:05.739Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607800165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148152024-02-15T03:04:05.739Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607816083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148162024-02-15T03:04:05.739Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.60783697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148172024-02-15T03:04:05.739Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607853928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148182024-02-15T03:04:05.739Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607878615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148192024-02-15T03:04:05.739Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607893383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148202024-02-15T03:04:05.739Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607912711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148212024-02-15T03:04:05.739Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607937697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148222024-02-15T03:04:05.739Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.607960354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148232024-02-15T03:04:05.739Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608091918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148242024-02-15T03:04:05.739Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608133173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148252024-02-15T03:04:05.739Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:04.608335447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148262024-02-15T03:04:05.739Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:04.60839284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148272024-02-15T03:04:05.739Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:04.608427946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148282024-02-15T03:04:05.739Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608466661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148292024-02-15T03:04:05.739Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608493798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148302024-02-15T03:04:05.740Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608547681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148312024-02-15T03:04:05.740Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608843754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148322024-02-15T03:04:05.740Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608893728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148332024-02-15T03:04:05.740Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608918214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148342024-02-15T03:04:05.740Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.608941302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148352024-02-15T03:04:05.740Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.609413643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148362024-02-15T03:04:05.740Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.609540687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148372024-02-15T03:04:05.740Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.609983211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148382024-02-15T03:04:05.740Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.610021746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148392024-02-15T03:04:05.740Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.610038934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148402024-02-15T03:04:05.740Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.610100746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148412024-02-15T03:04:05.740Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.610129813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148422024-02-15T03:04:05.740Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:04.610144791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148432024-02-15T03:04:06.727Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.610059765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148442024-02-15T03:04:06.768Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.610274478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148452024-02-15T03:04:06.768Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.611225819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148462024-02-15T03:04:06.768Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.611262794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148472024-02-15T03:04:06.768Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.611738324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148482024-02-15T03:04:06.768Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612300714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148492024-02-15T03:04:06.769Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612358946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148502024-02-15T03:04:06.769Z{"msg":"[0]R flush_numbers[0..12]: [219, 217, 220, 215, 210, 214, 220, 206, 215, 214, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612402511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148512024-02-15T03:04:06.769Z{"msg":"[0]R generation[0..12]: [132, 132, 133, 130, 127, 129, 133, 125, 130, 129, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612423298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148522024-02-15T03:04:06.769Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612451055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148532024-02-15T03:04:06.769Z{"msg":"[1]R flush_numbers[0..12]: [219, 217, 220, 215, 210, 214, 220, 206, 215, 214, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612469593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148542024-02-15T03:04:06.769Z{"msg":"[1]R generation[0..12]: [132, 132, 133, 130, 127, 129, 133, 125, 130, 129, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612495549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
148552024-02-15T03:04:06.769Z390cc217-c828-41ad-87d5-ec9a3cc58897"}
148562024-02-15T03:04:06.769Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612518546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148572024-02-15T03:04:06.769Z{"msg":"[2]R flush_numbers[0..12]: [219, 217, 220, 215, 210, 214, 220, 206, 215, 214, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612545473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148582024-02-15T03:04:06.769Z{"msg":"[2]R generation[0..12]: [132, 132, 133, 130, 127, 129, 133, 125, 130, 129, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612563401Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148592024-02-15T03:04:06.769Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612596127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148602024-02-15T03:04:06.769Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612609755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148612024-02-15T03:04:06.769Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612630852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148622024-02-15T03:04:06.769Z{"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612644281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148632024-02-15T03:04:06.770Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612663958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148642024-02-15T03:04:06.770Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612677917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148652024-02-15T03:04:06.770Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612696554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148662024-02-15T03:04:06.770Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612710342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148672024-02-15T03:04:06.770Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.61273005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148682024-02-15T03:04:06.771Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612743918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148692024-02-15T03:04:06.771Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612762856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148702024-02-15T03:04:06.771Z{"msg":"ae624513-b32a-4ddd-8afc-443e3dfeceeb is now active with session: 390cc217-c828-41ad-87d5-ec9a3cc58897","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612775924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148712024-02-15T03:04:06.772Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.612794162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674,"session_id":"390cc217-c828-41ad-87d5-ec9a3cc58897"}
148722024-02-15T03:04:06.772Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:05.61280693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674}
148732024-02-15T03:04:06.772ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148742024-02-15T03:04:06.772ZLoading write count information from file "/tmp/test_repair_verify.data"
148752024-02-15T03:04:06.772ZRun Repair workload
148762024-02-15T03:04:06.772Z01/30 Write block 96 len 16384 data: 13 15 15 13
148772024-02-15T03:04:06.772Z02/30 Write block 509 len 8192 data: 12 11
148782024-02-15T03:04:06.772Z03/30 Read block 97 len 16384
148792024-02-15T03:04:06.772Z04/30 Write block 440 len 12288 data: 12 12 13
148802024-02-15T03:04:06.773Z05/30 Read block 537 len 20480
148812024-02-15T03:04:06.773Z06/30 Flush
148822024-02-15T03:04:06.773Z07/30 Write block 559 len 36864 data: 15 14 10 10 10 10 12 7 7
148832024-02-15T03:04:06.773Z08/30 Read block 68 len 16384
148842024-02-15T03:04:06.774Z09/30 Write block 377 len 20480 data: 11 12 9 9 11
148852024-02-15T03:04:06.774Z10/30 Write block 535 len 36864 data: 10 7 10 9 10 8 10 12 13
148862024-02-15T03:04:06.774Z11/30 Write block 342 len 24576 data: 8 6 9 10 12 11
148872024-02-15T03:04:06.774Z12/30 Read block 3 len 24576
148882024-02-15T03:04:06.774Z13/30 Read block 544 len 32768
148892024-02-15T03:04:06.774Z14/30 Write block 398 len 40960 data: 9 11 9 8 7 6 6 11 11 13
148902024-02-15T03:04:06.774Z15/30 Read block 261 len 36864
148912024-02-15T03:04:06.774Z16/30 Write block 192 len 16384 data: 11 14 14 15
148922024-02-15T03:04:06.774Z17/30 Read block 322 len 32768
148932024-02-15T03:04:06.774Z18/30 Write block 143 len 36864 data: 9 12 11 11 11 8 8 8 8
148942024-02-15T03:04:06.774Z19/30 Write block 242 len 4096 data: 15
148952024-02-15T03:04:06.774Z20/30 Read block 263 len 24576
148962024-02-15T03:04:06.775Z21/30 Write block 353 len 40960 data: 13 12 12 11 11 13 14 11 11 12
148972024-02-15T03:04:06.775Z22/30 Flush
148982024-02-15T03:04:06.809Z23/30 Write block 142 len 16384 data: 12 10 13 12
148992024-02-15T03:04:06.810Z24/30 Flush
149002024-02-15T03:04:06.810Z25/30 Write block 540 len 28672 data: 9 11 13 14 7 6 8
149012024-02-15T03:04:06.810Z26/30 Write block 187 len 20480 data: 10 10 13 12 11
149022024-02-15T03:04:06.810Z27/30 Write block 75 len 16384 data: 7 7 6 5
149032024-02-15T03:04:06.810Z28/30 Write block 499 len 28672 data: 12 12 12 13 14 13 11
149042024-02-15T03:04:06.810Z29/30 Read block 184 len 16384
149052024-02-15T03:04:06.810Z30/30 Read block 367 len 8192
149062024-02-15T03:04:06.810Z----------------------------------------------------------------
149072024-02-15T03:04:06.810Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
149082024-02-15T03:04:06.810ZStates: Active Active Active
149092024-02-15T03:04:06.810ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149102024-02-15T03:04:06.810Z 1 Acked 1000 Write 4 Done Done Done false
149112024-02-15T03:04:06.810Z 2 Acked 1001 Write 2 Done Done Done false
149122024-02-15T03:04:06.811Z 3 Acked 1002 Read 4 Done Done Done false
149132024-02-15T03:04:06.811Z 4 Acked 1003 Write 3 Done Done Done false
149142024-02-15T03:04:06.811Z 5 Acked 1004 Read 5 Done Done Done false
149152024-02-15T03:04:06.812Z 6 Acked 1005 Flush 0 Sent Done Done false
149162024-02-15T03:04:06.812Z 7 Acked 1006 Write 9 Sent Done Done false
149172024-02-15T03:04:06.812Z 8 Acked 1007 Read 4 Sent Done Done false
149182024-02-15T03:04:06.812Z 9 Acked 1008 Write 5 Sent Done Done false
149192024-02-15T03:04:06.812Z 10 Acked 1009 Write 9 Sent Done Done false
149202024-02-15T03:04:06.812Z 11 Acked 1010 Write 6 Sent Done Done false
149212024-02-15T03:04:06.812Z 12 Acked 1011 Read 6 Sent Done Done false
149222024-02-15T03:04:06.812Z 13 Acked 1012 Read 8 Sent Done Done false
149232024-02-15T03:04:06.812Z 14 Acked 1013 Write 10 Sent Done Done false
149242024-02-15T03:04:06.812Z 15 Acked 1014 Read 9 Sent Done Done false
149252024-02-15T03:04:06.812Z 16 Acked 1015 Write 4 Sent Done Done false
149262024-02-15T03:04:06.812Z 17 Acked 1016 Read 8 Sent Done Done false
149272024-02-15T03:04:06.813Z 18 Acked 1017 Write 9 Sent Done Done false
149282024-02-15T03:04:06.813Z 19 Acked 1018 Write 1 Sent Done Done false
149292024-02-15T03:04:06.813Z 20 Acked 1019 Read 6 Sent Done Done false
149302024-02-15T03:04:06.813Z 21 Acked 1020 Write 10 Sent Done Done false
149312024-02-15T03:04:06.813Z 22 Acked 1021 Flush 0 Sent Done Done false
149322024-02-15T03:04:06.813Z 23 Acked 1022 Write 4 Sent Done Done false
149332024-02-15T03:04:06.813Z 24 Acked 1023 Flush 0 Sent Done Done false
149342024-02-15T03:04:06.814Z 25 Acked 1024 Write 7 Sent Done Done false
149352024-02-15T03:04:06.814Z 26 Acked 1025 Write 5 Sent Done Done false
149362024-02-15T03:04:06.814Z 27 Acked 1026 Write 4 Sent Done Done false
149372024-02-15T03:04:06.814Z 28 Acked 1027 Write 7 Sent Done Done false
149382024-02-15T03:04:06.814Z 29 Acked 1028 Read 4 Sent Done Done false
149392024-02-15T03:04:06.814Z 30 Acked 1029 Read 2 Sent Sent Done false
149402024-02-15T03:04:06.814Z STATES DS:0 DS:1 DS:2 TOTAL
149412024-02-15T03:04:06.814Z New 0 0 0 0
149422024-02-15T03:04:06.814Z Sent 25 1 0 26
149432024-02-15T03:04:06.814Z Done 5 29 30 64
149442024-02-15T03:04:06.814Z Skipped 0 0 0 0
149452024-02-15T03:04:06.814Z Error 0 0 0 0
149462024-02-15T03:04:06.814ZLast Flush: 0 1023 1023
149472024-02-15T03:04:06.814ZDownstairs last five completed:
149482024-02-15T03:04:06.814ZUpstairs last five completed: 30 29 28 27 26
149492024-02-15T03:04:06.814Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:05.651421762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1674}
149502024-02-15T03:04:06.815ZWrote out file "/tmp/test_repair_verify.data"
149512024-02-15T03:04:06.815Z
149522024-02-15T03:04:06.815ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149532024-02-15T03:04:06.815ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149542024-02-15T03:04:06.815Z 0 000-019 132 132 132 219 219 219 F F F
149552024-02-15T03:04:06.815Z 1 020-039 132 132 132 217 217 217 F F F
149562024-02-15T03:04:06.815Z 2 040-059 133 133 133 220 220 220 F F F
149572024-02-15T03:04:06.815Z 3 060-079 130 130 130 215 215 215 F T T <---
149582024-02-15T03:04:06.815Z 4 080-099 127 134 134 210 221 221 T F F <---
149592024-02-15T03:04:06.815Z 5 100-119 129 129 129 214 214 214 F F F
149602024-02-15T03:04:06.815Z 6 120-139 133 133 133 220 220 220 F F F
149612024-02-15T03:04:06.815Z 7 140-159 125 134 134 206 223 223 F F F <---
149622024-02-15T03:04:06.815Z 8 160-179 130 130 130 215 215 215 F F F
149632024-02-15T03:04:06.815Z 9 180-199 129 134 134 214 222 222 F T T <---
149642024-02-15T03:04:06.815Z 10 200-219 130 130 130 215 215 215 F F F
149652024-02-15T03:04:06.815Z 11 220-239 132 132 132 219 219 219 F F F
149662024-02-15T03:04:06.816Z 12 240-259 133 134 134 220 222 222 F F F <---
149672024-02-15T03:04:06.816Z 13 260-279 133 133 133 220 220 220 F F F
149682024-02-15T03:04:06.816Z 14 280-299 130 130 130 215 215 215 F F F
149692024-02-15T03:04:06.816Z 15 300-319 128 128 128 213 213 213 F F F
149702024-02-15T03:04:06.816Z 16 320-339 132 132 132 219 219 219 F F F
149712024-02-15T03:04:06.816Z 17 340-359 130 134 134 215 222 222 F F F <---
149722024-02-15T03:04:06.816Z 18 360-379 133 134 134 220 222 222 F F F <---
149732024-02-15T03:04:06.816Z 19 380-399 130 134 134 215 222 222 F F F <---
149742024-02-15T03:04:06.816Z 20 400-419 132 134 134 216 222 222 F F F <---
149752024-02-15T03:04:06.816Z 21 420-439 132 132 132 216 216 216 F F F
149762024-02-15T03:04:06.816Z 22 440-459 125 134 134 206 221 221 T F F <---
149772024-02-15T03:04:06.816Z 23 460-479 132 132 132 219 219 219 F F F
149782024-02-15T03:04:06.816Z 24 480-499 133 133 133 220 220 220 F T T <---
149792024-02-15T03:04:06.817Z 25 500-519 133 134 134 220 221 221 T T T <---
149802024-02-15T03:04:06.817Z 26 520-539 132 134 134 219 222 222 F F F <---
149812024-02-15T03:04:06.817Z 27 540-559 119 134 134 200 222 222 F T T <---
149822024-02-15T03:04:06.817Z 28 560-579 132 134 134 218 222 222 F F F <---
149832024-02-15T03:04:06.818Z 29 580-599 130 130 130 215 215 215 F F F
149842024-02-15T03:04:06.818ZMax gen: 134, Max flush: 223
149852024-02-15T03:04:06.818ZError: Difference in extent metadata found!
149862024-02-15T03:04:06.818ZA Difference in extent metadata is expected here
149872024-02-15T03:04:06.818ZOn loop 67
149882024-02-15T03:04:06.818Z
149892024-02-15T03:04:07.255ZVerifying data now
149902024-02-15T03:04:12.527ZNo change after verify
149912024-02-15T03:04:12.528ZLoop: 67 Downstairs dump after verify (and repair):
149922024-02-15T03:04:12.542ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149932024-02-15T03:04:12.542Z 0 000-019 132 132 132 219 219 219 F F F
149942024-02-15T03:04:12.542Z 1 020-039 132 132 132 217 217 217 F F F
149952024-02-15T03:04:12.542Z 2 040-059 133 133 133 220 220 220 F F F
149962024-02-15T03:04:12.542Z 3 060-079 135 135 135 224 224 224 F F F
149972024-02-15T03:04:12.543Z 4 080-099 134 134 134 221 221 221 F F F
149982024-02-15T03:04:12.543Z 5 100-119 129 129 129 214 214 214 F F F
149992024-02-15T03:04:12.543Z 6 120-139 133 133 133 220 220 220 F F F
150002024-02-15T03:04:12.559Z 7 140-159 134 134 134 223 223 223 F F F
150012024-02-15T03:04:12.559Z 8 160-179 130 130 130 215 215 215 F F F
150022024-02-15T03:04:12.559Z 9 180-199 135 135 135 224 224 224 F F F
150032024-02-15T03:04:12.559Z 10 200-219 130 130 130 215 215 215 F F F
150042024-02-15T03:04:12.559Z 11 220-239 132 132 132 219 219 219 F F F
150052024-02-15T03:04:12.559Z 12 240-259 134 134 134 222 222 222 F F F
150062024-02-15T03:04:12.559Z 13 260-279 133 133 133 220 220 220 F F F
150072024-02-15T03:04:12.560Z 14 280-299 130 130 130 215 215 215 F F F
150082024-02-15T03:04:12.560Z 15 300-319 128 128 128 213 213 213 F F F
150092024-02-15T03:04:12.560Z 16 320-339 132 132 132 219 219 219 F F F
150102024-02-15T03:04:12.560Z 17 340-359 134 134 134 222 222 222 F F F
150112024-02-15T03:04:12.560Z 18 360-379 134 134 134 222 222 222 F F F
150122024-02-15T03:04:12.560Z 19 380-399 134 134 134 222 222 222 F F F
150132024-02-15T03:04:12.560Z 20 400-419 134 134 134 222 222 222 F F F
150142024-02-15T03:04:12.561Z 21 420-439 132 132 132 216 216 216 F F F
150152024-02-15T03:04:12.561Z 22 440-459 134 134 134 221 221 221 F F F
150162024-02-15T03:04:12.561Z 23 460-479 132 132 132 219 219 219 F F F
150172024-02-15T03:04:12.561Z 24 480-499 135 135 135 224 224 224 F F F
150182024-02-15T03:04:12.561Z 25 500-519 135 135 135 224 224 224 F F F
150192024-02-15T03:04:12.561Z 26 520-539 134 134 134 222 222 222 F F F
150202024-02-15T03:04:12.561Z 27 540-559 135 135 135 224 224 224 F F F
150212024-02-15T03:04:12.561Z 28 560-579 134 134 134 222 222 222 F F F
150222024-02-15T03:04:12.562Z 29 580-599 130 130 130 215 215 215 F F F
150232024-02-15T03:04:12.562ZMax gen: 135, Max flush: 224
150242024-02-15T03:04:12.562Z
150252024-02-15T03:04:12.562ZBegin loop 68
150262024-02-15T03:04:12.563ZDownstairs to restart: 0
150272024-02-15T03:04:12.571Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.461973348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682}
150282024-02-15T03:04:12.571Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462309346Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682}
150292024-02-15T03:04:12.571ZCrucible runtime is spawned
150302024-02-15T03:04:12.571Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462359279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682}
150312024-02-15T03:04:12.571Z{"msg":"Crucible 451cfca1-d8ca-4109-be52-9466d6a7429d has session id: 7de06f62-133c-4fe9-9d95-9b55b832a631","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462401254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150322024-02-15T03:04:12.571Z{"msg":"Upstairs opts: Upstairs UUID: 451cfca1-d8ca-4109-be52-9466d6a7429d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462461957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150332024-02-15T03:04:12.571Z{"msg":"Crucible stats registered with UUID: 451cfca1-d8ca-4109-be52-9466d6a7429d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462505601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150342024-02-15T03:04:12.571Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462534837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682}
150352024-02-15T03:04:12.571Z{"msg":"451cfca1-d8ca-4109-be52-9466d6a7429d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462568683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150362024-02-15T03:04:12.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462614317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150372024-02-15T03:04:12.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462642454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150382024-02-15T03:04:12.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462685549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150392024-02-15T03:04:12.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462724653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150402024-02-15T03:04:12.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.46275298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150412024-02-15T03:04:12.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462842439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150422024-02-15T03:04:12.572Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462891573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150432024-02-15T03:04:12.572Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462931138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150442024-02-15T03:04:12.572Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462959464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150452024-02-15T03:04:12.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.462985951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150462024-02-15T03:04:12.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.463029425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150472024-02-15T03:04:12.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.463058292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150482024-02-15T03:04:12.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.463613812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150492024-02-15T03:04:12.572Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.463661136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150502024-02-15T03:04:12.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.463693292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150512024-02-15T03:04:12.572Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.463754664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150522024-02-15T03:04:12.572Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464212797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150532024-02-15T03:04:12.572Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464356859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150542024-02-15T03:04:12.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464702066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150552024-02-15T03:04:12.573Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464770417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150562024-02-15T03:04:12.573Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464855007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150572024-02-15T03:04:12.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464892672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150582024-02-15T03:04:12.573Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464912989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150592024-02-15T03:04:12.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464944066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150602024-02-15T03:04:12.573Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464962503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150612024-02-15T03:04:12.573Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.464993189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150622024-02-15T03:04:12.573Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465330627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150632024-02-15T03:04:12.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465779411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150642024-02-15T03:04:12.573Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465812427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150652024-02-15T03:04:12.573Z{"msg":"[0]R flush_numbers[0..12]: [219, 217, 220, 224, 221, 214, 220, 223, 215, 224, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465827885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150662024-02-15T03:04:12.573Z{"msg":"[0]R generation[0..12]: [132, 132, 133, 135, 134, 129, 133, 134, 130, 135, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465846333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150672024-02-15T03:04:12.573Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465876969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150682024-02-15T03:04:12.573Z{"msg":"[1]R flush_numbers[0..12]: [219, 217, 220, 224, 221, 214, 220, 223, 215, 224, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465896736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150692024-02-15T03:04:12.573Z{"msg":"[1]R generation[0..12]: [132, 132, 133, 135, 134, 129, 133, 134, 130, 135, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465926462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150702024-02-15T03:04:12.573Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.46594225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150712024-02-15T03:04:12.573ZWait for a query_work_queue command to finish before sending IO
150722024-02-15T03:04:12.573Z{"msg":"[2]R flush_numbers[0..12]: [219, 217, 220, 224, 221, 214, 220, 223, 215, 224, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.465975136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150732024-02-15T03:04:12.573Z{"msg":"[2]R generation[0..12]: [132, 132, 133, 135, 134, 129, 133, 134, 130, 135, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466007442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150742024-02-15T03:04:12.573Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466042688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150752024-02-15T03:04:12.574Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466072834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150762024-02-15T03:04:12.574Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466113489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150772024-02-15T03:04:12.574Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466143515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150782024-02-15T03:04:12.574Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466162723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150792024-02-15T03:04:12.574Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466176811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150802024-02-15T03:04:12.574Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466201108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150812024-02-15T03:04:12.574Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466215666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150822024-02-15T03:04:12.574Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466236133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150832024-02-15T03:04:12.574Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466249912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150842024-02-15T03:04:12.574Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.46626234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150852024-02-15T03:04:12.574Z{"msg":"451cfca1-d8ca-4109-be52-9466d6a7429d is now active with session: 7de06f62-133c-4fe9-9d95-9b55b832a631","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466276368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150862024-02-15T03:04:12.574Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466289127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682,"session_id":"7de06f62-133c-4fe9-9d95-9b55b832a631"}
150872024-02-15T03:04:12.574Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:11.466302155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682}
150882024-02-15T03:04:12.574ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150892024-02-15T03:04:12.574ZLoading write count information from file "/tmp/test_repair_verify.data"
150902024-02-15T03:04:12.574ZRun Repair workload
150912024-02-15T03:04:12.574Z01/30 Write block 45 len 4096 data: 13
150922024-02-15T03:04:12.578Z02/30 Read block 250 len 40960
150932024-02-15T03:04:12.578Z03/30 Read block 307 len 36864
150942024-02-15T03:04:12.578Z04/30 Write block 287 len 28672 data: 14 10 12 19 17 16 14
150952024-02-15T03:04:12.578Z05/30 Write block 544 len 28672 data: 8 7 9 8 9 8 7
150962024-02-15T03:04:12.578Z06/30 Read block 588 len 4096
150972024-02-15T03:04:12.578Z07/30 Write block 560 len 36864 data: 15 11 11 11 11 13 8 8 7
150982024-02-15T03:04:12.578Z08/30 Write block 305 len 16384 data: 16 13 12 12
150992024-02-15T03:04:12.578Z09/30 Read block 4 len 20480
151002024-02-15T03:04:12.578Z10/30 Read block 438 len 16384
151012024-02-15T03:04:12.578Z11/30 Write block 528 len 12288 data: 11 13 13
151022024-02-15T03:04:12.578Z12/30 Read block 196 len 12288
151032024-02-15T03:04:12.583Z13/30 Write block 256 len 24576 data: 9 11 11 13 10 8
151042024-02-15T03:04:12.583Z14/30 Read block 59 len 8192
151052024-02-15T03:04:12.583Z15/30 Write block 38 len 32768 data: 12 10 8 10 11 11 13 14
151062024-02-15T03:04:12.583Z16/30 Write block 577 len 40960 data: 17 14 12 13 9 11 10 10 11 11
151072024-02-15T03:04:12.583Z17/30 Read block 348 len 28672
151082024-02-15T03:04:12.593Z18/30 Write block 297 len 8192 data: 10 10
151092024-02-15T03:04:12.593Z19/30 Read block 180 len 28672
151102024-02-15T03:04:12.593Z20/30 Write block 331 len 36864 data: 15 14 14 17 16 14 14 14 10
151112024-02-15T03:04:12.593Z21/30 Read block 333 len 4096
151122024-02-15T03:04:12.593Z22/30 Read block 19 len 40960
151132024-02-15T03:04:12.600Z23/30 Read block 274 len 4096
151142024-02-15T03:04:12.600Z24/30 Read block 306 len 16384
151152024-02-15T03:04:12.600Z25/30 Read block 394 len 4096
151162024-02-15T03:04:12.600Z26/30 Write block 45 len 20480 data: 15 11 13 11 10
151172024-02-15T03:04:12.600Z27/30 Write block 69 len 16384 data: 8 8 9 8
151182024-02-15T03:04:12.600Z28/30 Read block 580 len 12288
151192024-02-15T03:04:12.600Z29/30 Write block 566 len 16384 data: 9 9 8 7
151202024-02-15T03:04:12.600Z30/30 Read block 10 len 24576
151212024-02-15T03:04:12.600Z----------------------------------------------------------------
151222024-02-15T03:04:12.600Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
151232024-02-15T03:04:12.600ZStates: Active Active Active
151242024-02-15T03:04:12.601ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151252024-02-15T03:04:12.601Z 1 Acked 1000 Write 1 Sent Done Done false
151262024-02-15T03:04:12.601Z 2 Acked 1001 Read 10 Sent Done Done false
151272024-02-15T03:04:12.601Z 3 Acked 1002 Read 9 Sent Done Done false
151282024-02-15T03:04:12.601Z 4 Acked 1003 Write 7 Sent Done Done false
151292024-02-15T03:04:12.601Z 5 Acked 1004 Write 7 Sent Done Done false
151302024-02-15T03:04:12.601Z 6 Acked 1005 Read 1 Sent Done Done false
151312024-02-15T03:04:12.601Z 7 Acked 1006 Write 9 Sent Done Done false
151322024-02-15T03:04:12.601Z 8 Acked 1007 Write 4 Sent Done Done false
151332024-02-15T03:04:12.601Z 9 Acked 1008 Read 5 Sent Done Done false
151342024-02-15T03:04:12.601Z 10 Acked 1009 Read 4 Sent Done Done false
151352024-02-15T03:04:12.601Z 11 Acked 1010 Write 3 Sent Done Done false
151362024-02-15T03:04:12.601Z 12 Acked 1011 Read 3 Sent Done Done false
151372024-02-15T03:04:12.601Z 13 Acked 1012 Write 6 Sent Done Done false
151382024-02-15T03:04:12.601Z 14 Acked 1013 Read 2 Sent Done Done false
151392024-02-15T03:04:12.601Z 15 Acked 1014 Write 8 Sent Done Done false
151402024-02-15T03:04:12.601Z 16 Acked 1015 Write 10 Sent Done Done false
151412024-02-15T03:04:12.601Z 17 Acked 1016 Read 7 Sent Done Done false
151422024-02-15T03:04:12.601Z 18 Acked 1017 Write 2 Sent Done Done false
151432024-02-15T03:04:12.602Z 19 Acked 1018 Read 7 Sent Done Done false
151442024-02-15T03:04:12.602Z 20 Acked 1019 Write 9 Sent Done Done false
151452024-02-15T03:04:12.602Z 21 Acked 1020 Read 1 Sent Done Done false
151462024-02-15T03:04:12.602Z 22 Acked 1021 Read 10 Sent Done Done false
151472024-02-15T03:04:12.602Z 23 Acked 1022 Read 1 Sent Done Done false
151482024-02-15T03:04:12.602Z 24 Acked 1023 Read 4 Sent Done Done false
151492024-02-15T03:04:12.602Z 25 Acked 1024 Read 1 Sent Done Done false
151502024-02-15T03:04:12.602Z 26 Acked 1025 Write 5 Sent Done Done false
151512024-02-15T03:04:12.602Z 27 Acked 1026 Write 4 Sent Done Done false
151522024-02-15T03:04:12.602Z 28 Acked 1027 Read 3 Sent Done Done false
151532024-02-15T03:04:12.602Z 29 Acked 1028 Write 4 Sent Done Done false
151542024-02-15T03:04:12.602Z 30 Acked 1029 Read 6 Sent Done Sent false
151552024-02-15T03:04:12.602Z STATES DS:0 DS:1 DS:2 TOTAL
151562024-02-15T03:04:12.602Z New 0 0 0 0
151572024-02-15T03:04:12.602Z Sent 30 0 1 31
151582024-02-15T03:04:12.602Z Done 0 30 29 59
151592024-02-15T03:04:12.602Z Skipped 0 0 0 0
151602024-02-15T03:04:12.602Z Error 0 0 0 0
151612024-02-15T03:04:12.602ZLast Flush: 0 0 0
151622024-02-15T03:04:12.603ZDownstairs last five completed:
151632024-02-15T03:04:12.603ZUpstairs last five completed: 30 29 28 27 26
151642024-02-15T03:04:12.603Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:11.49475686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1682}
151652024-02-15T03:04:12.603ZWrote out file "/tmp/test_repair_verify.data"
151662024-02-15T03:04:12.603Z
151672024-02-15T03:04:12.607ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151682024-02-15T03:04:12.623ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151692024-02-15T03:04:12.623Z 0 000-019 132 132 132 219 219 219 F F F
151702024-02-15T03:04:12.623Z 1 020-039 132 132 132 217 217 217 F T T <---
151712024-02-15T03:04:12.623Z 2 040-059 133 133 133 220 220 220 F T T <---
151722024-02-15T03:04:12.623Z 3 060-079 135 135 135 224 224 224 F T T <---
151732024-02-15T03:04:12.623Z 4 080-099 134 134 134 221 221 221 F F F
151742024-02-15T03:04:12.624Z 5 100-119 129 129 129 214 214 214 F F F
151752024-02-15T03:04:12.624Z 6 120-139 133 133 133 220 220 220 F F F
151762024-02-15T03:04:12.624Z 7 140-159 134 134 134 223 223 223 F F F
151772024-02-15T03:04:12.624Z 8 160-179 130 130 130 215 215 215 F F F
151782024-02-15T03:04:12.624Z 9 180-199 135 135 135 224 224 224 F F F
151792024-02-15T03:04:12.624Z 10 200-219 130 130 130 215 215 215 F F F
151802024-02-15T03:04:12.624Z 11 220-239 132 132 132 219 219 219 F F F
151812024-02-15T03:04:12.624Z 12 240-259 134 134 134 222 222 222 F T T <---
151822024-02-15T03:04:12.624Z 13 260-279 133 133 133 220 220 220 F T T <---
151832024-02-15T03:04:12.624Z 14 280-299 130 130 130 215 215 215 F T T <---
151842024-02-15T03:04:12.628Z 15 300-319 128 128 128 213 213 213 F T T <---
151852024-02-15T03:04:12.628Z 16 320-339 132 132 132 219 219 219 F T T <---
151862024-02-15T03:04:12.628Z 17 340-359 134 134 134 222 222 222 F F F
151872024-02-15T03:04:12.628Z 18 360-379 134 134 134 222 222 222 F F F
151882024-02-15T03:04:12.628Z 19 380-399 134 134 134 222 222 222 F F F
151892024-02-15T03:04:12.628Z 20 400-419 134 134 134 222 222 222 F F F
151902024-02-15T03:04:12.628Z 21 420-439 132 132 132 216 216 216 F F F
151912024-02-15T03:04:12.628Z 22 440-459 134 134 134 221 221 221 F F F
151922024-02-15T03:04:12.628Z 23 460-479 132 132 132 219 219 219 F F F
151932024-02-15T03:04:12.628Z 24 480-499 135 135 135 224 224 224 F F F
151942024-02-15T03:04:12.628Z 25 500-519 135 135 135 224 224 224 F F F
151952024-02-15T03:04:12.628Z 26 520-539 134 134 134 222 222 222 F T T <---
151962024-02-15T03:04:12.628Z 27 540-559 135 135 135 224 224 224 F T T <---
151972024-02-15T03:04:12.629Z 28 560-579 134 134 134 222 222 222 F T T <---
151982024-02-15T03:04:12.629Z 29 580-599 130 130 130 215 215 215 F T T <---
151992024-02-15T03:04:12.629ZMax gen: 135, Max flush: 224
152002024-02-15T03:04:12.629ZError: Difference in extent metadata found!
152012024-02-15T03:04:12.629ZA Difference in extent metadata is expected here
152022024-02-15T03:04:12.629ZOn loop 68
152032024-02-15T03:04:12.629Z
152042024-02-15T03:04:12.633ZVerifying data now
152052024-02-15T03:04:17.282ZNo change after verify
152062024-02-15T03:04:17.282ZLoop: 68 Downstairs dump after verify (and repair):
152072024-02-15T03:04:17.340ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152082024-02-15T03:04:17.340Z 0 000-019 132 132 132 219 219 219 F F F
152092024-02-15T03:04:17.340Z 1 020-039 136 136 136 225 225 225 F F F
152102024-02-15T03:04:17.340Z 2 040-059 136 136 136 225 225 225 F F F
152112024-02-15T03:04:17.341Z 3 060-079 136 136 136 225 225 225 F F F
152122024-02-15T03:04:17.341Z 4 080-099 134 134 134 221 221 221 F F F
152132024-02-15T03:04:17.341Z 5 100-119 129 129 129 214 214 214 F F F
152142024-02-15T03:04:17.341Z 6 120-139 133 133 133 220 220 220 F F F
152152024-02-15T03:04:17.341Z 7 140-159 134 134 134 223 223 223 F F F
152162024-02-15T03:04:17.341Z 8 160-179 130 130 130 215 215 215 F F F
152172024-02-15T03:04:17.341Z 9 180-199 135 135 135 224 224 224 F F F
152182024-02-15T03:04:17.341Z 10 200-219 130 130 130 215 215 215 F F F
152192024-02-15T03:04:17.341Z 11 220-239 132 132 132 219 219 219 F F F
152202024-02-15T03:04:17.342Z 12 240-259 136 136 136 225 225 225 F F F
152212024-02-15T03:04:17.342Z 13 260-279 136 136 136 225 225 225 F F F
152222024-02-15T03:04:17.342Z 14 280-299 136 136 136 225 225 225 F F F
152232024-02-15T03:04:17.342Z 15 300-319 136 136 136 225 225 225 F F F
152242024-02-15T03:04:17.342Z 16 320-339 136 136 136 225 225 225 F F F
152252024-02-15T03:04:17.342Z 17 340-359 134 134 134 222 222 222 F F F
152262024-02-15T03:04:17.342Z 18 360-379 134 134 134 222 222 222 F F F
152272024-02-15T03:04:17.343Z 19 380-399 134 134 134 222 222 222 F F F
152282024-02-15T03:04:17.343Z 20 400-419 134 134 134 222 222 222 F F F
152292024-02-15T03:04:17.343Z 21 420-439 132 132 132 216 216 216 F F F
152302024-02-15T03:04:17.372Z 22 440-459 134 134 134 221 221 221 F F F
152312024-02-15T03:04:17.372Z 23 460-479 132 132 132 219 219 219 F F F
152322024-02-15T03:04:17.373Z 24 480-499 135 135 135 224 224 224 F F F
152332024-02-15T03:04:17.373Z 25 500-519 135 135 135 224 224 224 F F F
152342024-02-15T03:04:17.373Z 26 520-539 136 136 136 225 225 225 F F F
152352024-02-15T03:04:17.373Z 27 540-559 136 136 136 225 225 225 F F F
152362024-02-15T03:04:17.373Z 28 560-579 136 136 136 225 225 225 F F F
152372024-02-15T03:04:17.374Z 29 580-599 136 136 136 225 225 225 F F F
152382024-02-15T03:04:17.374ZMax gen: 136, Max flush: 225
152392024-02-15T03:04:17.374Z
152402024-02-15T03:04:17.374ZBegin loop 69
152412024-02-15T03:04:17.374ZDownstairs to restart: 0
152422024-02-15T03:04:17.375Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.210691678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690}
152432024-02-15T03:04:17.375Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211063641Z","hostname":"Crucible runtime is spawned
152442024-02-15T03:04:17.375Zip-10-150-1-55.us-west-2.compute.internal","pid":1690}
152452024-02-15T03:04:17.375Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211102726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690}
152462024-02-15T03:04:17.375Z{"msg":"Crucible f4be0d92-77d1-4649-bb97-f3eb2bc2a8b5 has session id: a2377108-b277-4755-96a8-54d95468d3e8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211131463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152472024-02-15T03:04:17.375Z{"msg":"Upstairs opts: Upstairs UUID: f4be0d92-77d1-4649-bb97-f3eb2bc2a8b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211183486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152482024-02-15T03:04:17.375Z{"msg":"Crucible stats registered with UUID: f4be0d92-77d1-4649-bb97-f3eb2bc2a8b5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211203923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152492024-02-15T03:04:17.375Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.21122929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690}
152502024-02-15T03:04:17.375Z{"msg":"f4be0d92-77d1-4649-bb97-f3eb2bc2a8b5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211250448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152512024-02-15T03:04:17.375Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211290892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152522024-02-15T03:04:17.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211347635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152532024-02-15T03:04:17.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211395619Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152542024-02-15T03:04:17.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211429475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152552024-02-15T03:04:17.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.21146737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152562024-02-15T03:04:17.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211503026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152572024-02-15T03:04:17.376Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211538492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152582024-02-15T03:04:17.376Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211578966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152592024-02-15T03:04:17.376Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211618591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152602024-02-15T03:04:17.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211643878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152612024-02-15T03:04:17.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211688993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152622024-02-15T03:04:17.376Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:16.211819696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152632024-02-15T03:04:17.376Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:16.211855102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152642024-02-15T03:04:17.377Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:16.211875099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152652024-02-15T03:04:17.377Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211898286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152662024-02-15T03:04:17.377Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.211932492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152672024-02-15T03:04:17.377Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.212044138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152682024-02-15T03:04:17.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.212206588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152692024-02-15T03:04:17.377Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.212244743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152702024-02-15T03:04:17.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.212261191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152712024-02-15T03:04:17.377Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.212314184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152722024-02-15T03:04:17.377Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.212803753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152732024-02-15T03:04:17.377Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.212919119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152742024-02-15T03:04:17.377Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.213359543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152752024-02-15T03:04:17.377Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.213396439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152762024-02-15T03:04:17.377Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.213413026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152772024-02-15T03:04:17.377Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.213460791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152782024-02-15T03:04:17.377Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.213481018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152792024-02-15T03:04:17.377Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:16.213495156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152802024-02-15T03:04:18.321Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.213284756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152812024-02-15T03:04:18.321Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.213376614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152822024-02-15T03:04:18.330Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.214317866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152832024-02-15T03:04:18.330Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.214369589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152842024-02-15T03:04:18.330Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.214931789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152852024-02-15T03:04:18.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215557551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152862024-02-15T03:04:18.330Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215613714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152872024-02-15T03:04:18.331Z{"msg":"[0]R flush_numbers[0..12]: [219, 225, 225, 225, 221, 214, 220, 223, 215, 224, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215650749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152882024-02-15T03:04:18.331Z{"msg":"[0]R generation[0..12]: [132, 136, 136, 136, 134, 129, 133, 134, 130, 135, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215682335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152892024-02-15T03:04:18.331Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215726439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152902024-02-15T03:04:18.331Z{"msg":"[1]R flush_numbers[0..12]: [219, 225, 225, 225, 221, 214, 220, 223, 215, 224, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215759965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152912024-02-15T03:04:18.331Z{"msg":"[1]R generation[0..12]: [132, 136, 136, 136, 134, 129, 133, 134, 130, 135, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215795911Z",Wait for a query_work_queue command to finish before sending IO
152922024-02-15T03:04:18.331Z"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152932024-02-15T03:04:18.331Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215891799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152942024-02-15T03:04:18.331Z{"msg":"[2]R flush_numbers[0..12]: [219, 225, 225, 225, 221, 214, 220, 223, 215, 224, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.215937313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152952024-02-15T03:04:18.332Z{"msg":"[2]R generation[0..12]: [132, 136, 136, 136, 134, 129, 133, 134, 130, 135, 130, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.21596538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152962024-02-15T03:04:18.332Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216001715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152972024-02-15T03:04:18.332Z{"msg":"Max found gen is 137","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216020813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152982024-02-15T03:04:18.332Z{"msg":"Generation requested: 138 >= found:137","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216046739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
152992024-02-15T03:04:18.332Z{"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216063567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153002024-02-15T03:04:18.332Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216096153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153012024-02-15T03:04:18.332Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.21611898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153022024-02-15T03:04:18.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216143577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153032024-02-15T03:04:18.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216161875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153042024-02-15T03:04:18.333Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216187042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153052024-02-15T03:04:18.333Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.21620432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153062024-02-15T03:04:18.333Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216220737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153072024-02-15T03:04:18.333Z{"msg":"f4be0d92-77d1-4649-bb97-f3eb2bc2a8b5 is now active with session: a2377108-b277-4755-96a8-54d95468d3e8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216237586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153082024-02-15T03:04:18.333Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216253093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690,"session_id":"a2377108-b277-4755-96a8-54d95468d3e8"}
153092024-02-15T03:04:18.333Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:17.216275751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690}
153102024-02-15T03:04:18.333ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153112024-02-15T03:04:18.333ZLoading write count information from file "/tmp/test_repair_verify.data"
153122024-02-15T03:04:18.334ZRun Repair workload
153132024-02-15T03:04:18.334Z01/30 Write block 183 len 28672 data: 8 10 12 13 11 11 14
153142024-02-15T03:04:18.334Z02/30 Read block 171 len 36864
153152024-02-15T03:04:18.334Z03/30 Read block 304 len 36864
153162024-02-15T03:04:18.334Z04/30 Read block 35 len 20480
153172024-02-15T03:04:18.334Z05/30 Write block 514 len 24576 data: 12 9 10 9 8 7
153182024-02-15T03:04:18.334Z06/30 Read block 520 len 4096
153192024-02-15T03:04:18.334Z07/30 Flush
153202024-02-15T03:04:18.339Z08/30 Write block 2 len 28672 data: 3 7 9 9 10 9 12
153212024-02-15T03:04:18.339Z09/30 Write block 199 len 16384 data: 10 11 9 6
153222024-02-15T03:04:18.339Z10/30 Flush
153232024-02-15T03:04:18.339Z11/30 Write block 208 len 8192 data: 9 8
153242024-02-15T03:04:18.339Z12/30 Write block 444 len 4096 data: 10
153252024-02-15T03:04:18.339Z13/30 Read block 380 len 4096
153262024-02-15T03:04:18.339Z14/30 Flush
153272024-02-15T03:04:18.339Z15/30 Write block 73 len 16384 data: 5 5 8 8
153282024-02-15T03:04:18.340Z16/30 Write block 340 len 36864 data: 8 8 9 7 10 11 13 12 10
153292024-02-15T03:04:18.340Z17/30 Write block 445 len 28672 data: 12 13 12 11 8 7 6
153302024-02-15T03:04:18.340Z18/30 Write block 60 len 36864 data: 11 11 12 12 12 9 9 9 8
153312024-02-15T03:04:18.340Z19/30 Write block 231 len 32768 data: 11 13 13 14 13 14 12 11
153322024-02-15T03:04:18.344Z20/30 Write block 375 len 24576 data: 8 10 12 13 10 10
153332024-02-15T03:04:18.344Z21/30 Write block 279 len 20480 data: 10 11 9 7 10
153342024-02-15T03:04:18.344Z22/30 Read block 370 len 40960
153352024-02-15T03:04:18.344Z23/30 Read block 574 len 24576
153362024-02-15T03:04:18.344Z24/30 Flush
153372024-02-15T03:04:18.348Z25/30 Write block 427 len 8192 data: 10 9
153382024-02-15T03:04:18.349Z26/30 Read block 511 len 32768
153392024-02-15T03:04:18.353Z27/30 Read block 137 len 8192
153402024-02-15T03:04:18.353Z28/30 Read block 485 len 36864
153412024-02-15T03:04:18.353Z29/30 Write block 33 len 32768 data: 13 13 15 13 12 13 11 9
153422024-02-15T03:04:18.353Z30/30 Read block 519 len 24576
153432024-02-15T03:04:18.354Z----------------------------------------------------------------
153442024-02-15T03:04:18.354Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
153452024-02-15T03:04:18.354ZStates: Active Active Active
153462024-02-15T03:04:18.354ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153472024-02-15T03:04:18.354Z 1 Acked 1000 Write 7 Done Done Done false
153482024-02-15T03:04:18.354Z 2 Acked 1001 Read 9 Done Done Done false
153492024-02-15T03:04:18.354Z 3 Acked 1002 Read 9 Done Done Done false
153502024-02-15T03:04:18.355Z 4 Acked 1003 Read 5 Done Done Done false
153512024-02-15T03:04:18.355Z 5 Acked 1004 Write 6 Sent Done Done false
153522024-02-15T03:04:18.355Z 6 Acked 1005 Read 1 Sent Done Done false
153532024-02-15T03:04:18.355Z 7 Acked 1006 Flush 0 Sent Done Done false
153542024-02-15T03:04:18.355Z 8 Acked 1007 Write 7 Sent Done Done false
153552024-02-15T03:04:18.355Z 9 Acked 1008 Write 4 Sent Done Done false
153562024-02-15T03:04:18.355Z 10 Acked 1009 Flush 0 Sent Done Done false
153572024-02-15T03:04:18.356Z 11 Acked 1010 Write 2 Sent Done Done false
153582024-02-15T03:04:18.356Z 12 Acked 1011 Write 1 Sent Done Done false
153592024-02-15T03:04:18.356Z 13 Acked 1012 Read 1 Sent Done Done false
153602024-02-15T03:04:18.356Z 14 Acked 1013 Flush 0 Sent Done Done false
153612024-02-15T03:04:18.356Z 15 Acked 1014 Write 4 Sent Done Done false
153622024-02-15T03:04:18.356Z 16 Acked 1015 Write 9 Sent Done Done false
153632024-02-15T03:04:18.356Z 17 Acked 1016 Write 7 Sent Done Done false
153642024-02-15T03:04:18.356Z 18 Acked 1017 Write 9 Sent Done Done false
153652024-02-15T03:04:18.356Z 19 Acked 1018 Write 8 Sent Done Done false
153662024-02-15T03:04:18.357Z 20 Acked 1019 Write 6 Sent Done Done false
153672024-02-15T03:04:18.357Z 21 Acked 1020 Write 5 Sent Done Done false
153682024-02-15T03:04:18.357Z 22 Acked 1021 Read 10 Sent Done Done false
153692024-02-15T03:04:18.357Z 23 Acked 1022 Read 6 Sent Done Done false
153702024-02-15T03:04:18.357Z 24 Acked 1023 Flush 0 Sent Done Done false
153712024-02-15T03:04:18.357Z 25 Acked 1024 Write 2 Sent Done Done false
153722024-02-15T03:04:18.357Z 26 Acked 1025 Read 8 Sent Done Done false
153732024-02-15T03:04:18.357Z 27 Acked 1026 Read 2 Sent Done Done false
153742024-02-15T03:04:18.357Z 28 Acked 1027 Read 9 Sent Done Done false
153752024-02-15T03:04:18.358Z 29 Acked 1028 Write 8 Sent Done Done false
153762024-02-15T03:04:18.358Z 30 Acked 1029 Read 6 Sent Done Done false
153772024-02-15T03:04:18.358Z STATES DS:0 DS:1 DS:2 TOTAL
153782024-02-15T03:04:18.358Z New 0 0 0 0
153792024-02-15T03:04:18.358Z Sent 26 0 0 26
153802024-02-15T03:04:18.358Z Done 4 30 30 64
153812024-02-15T03:04:18.358Z Skipped 0 0 0 0
153822024-02-15T03:04:18.358Z Error 0 0 0 0
153832024-02-15T03:04:18.358ZLast Flush: 0 1023 1023
153842024-02-15T03:04:18.358ZDownstairs last five completed:
153852024-02-15T03:04:18.358ZUpstairs last five completed: 30 29 28 27 26
153862024-02-15T03:04:18.358Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:17.250767239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1690}
153872024-02-15T03:04:18.358ZWrote out file "/tmp/test_repair_verify.data"
153882024-02-15T03:04:18.363Z
153892024-02-15T03:04:18.363ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153902024-02-15T03:04:18.380ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153912024-02-15T03:04:18.380Z 0 000-019 132 138 138 219 227 227 F F F <---
153922024-02-15T03:04:18.380Z 1 020-039 136 136 136 225 225 225 F T T <---
153932024-02-15T03:04:18.380Z 2 040-059 136 136 136 225 225 225 F T T <---
153942024-02-15T03:04:18.381Z 3 060-079 136 138 138 225 229 229 F F F <---
153952024-02-15T03:04:18.381Z 4 080-099 134 134 134 221 221 221 F F F
153962024-02-15T03:04:18.381Z 5 100-119 129 129 129 214 214 214 F F F
153972024-02-15T03:04:18.381Z 6 120-139 133 133 133 220 220 220 F F F
153982024-02-15T03:04:18.381Z 7 140-159 134 134 134 223 223 223 F F F
153992024-02-15T03:04:18.381Z 8 160-179 130 130 130 215 215 215 F F F
154002024-02-15T03:04:18.382Z 9 180-199 135 138 138 224 227 227 T F F <---
154012024-02-15T03:04:18.382Z 10 200-219 130 138 138 215 228 228 F F F <---
154022024-02-15T03:04:18.382Z 11 220-239 132 138 138 219 229 229 F F F <---
154032024-02-15T03:04:18.382Z 12 240-259 136 136 136 225 225 225 F F F
154042024-02-15T03:04:18.382Z 13 260-279 136 138 138 225 229 229 F F F <---
154052024-02-15T03:04:18.386Z 14 280-299 136 138 138 225 229 229 F F F <---
154062024-02-15T03:04:18.386Z 15 300-319 136 136 136 225 225 225 F F F
154072024-02-15T03:04:18.386Z 16 320-339 136 136 136 225 225 225 F F F
154082024-02-15T03:04:18.387Z 17 340-359 134 138 138 222 229 229 F F F <---
154092024-02-15T03:04:18.387Z 18 360-379 134 138 138 222 229 229 F F F <---
154102024-02-15T03:04:18.387Z 19 380-399 134 138 138 222 229 229 F F F <---
154112024-02-15T03:04:18.387Z 20 400-419 134 134 134 222 222 222 F F F
154122024-02-15T03:04:18.387Z 21 420-439 132 132 132 216 216 216 F T T <---
154132024-02-15T03:04:18.388Z 22 440-459 134 138 138 221 229 229 F F F <---
154142024-02-15T03:04:18.388Z 23 460-479 132 132 132 219 219 219 F F F
154152024-02-15T03:04:18.388Z 24 480-499 135 135 135 224 224 224 F F F
154162024-02-15T03:04:18.388Z 25 500-519 135 138 138 224 226 226 F F F <---
154172024-02-15T03:04:18.388Z 26 520-539 136 136 136 225 225 225 F F F
154182024-02-15T03:04:18.388Z 27 540-559 136 136 136 225 225 225 F F F
154192024-02-15T03:04:18.388Z 28 560-579 136 136 136 225 225 225 F F F
154202024-02-15T03:04:18.388Z 29 580-599 136 136 136 225 225 225 F F F
154212024-02-15T03:04:18.389ZMax gen: 138, Max flush: 229
154222024-02-15T03:04:18.389ZError: Difference in extent metadata found!
154232024-02-15T03:04:18.389ZA Difference in extent metadata is expected here
154242024-02-15T03:04:18.389ZOn loop 69
154252024-02-15T03:04:18.389Z
154262024-02-15T03:04:18.389ZVerifying data now
154272024-02-15T03:04:24.127ZNo change after verify
154282024-02-15T03:04:24.127ZLoop: 69 Downstairs dump after verify (and repair):
154292024-02-15T03:04:24.141ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154302024-02-15T03:04:24.141Z 0 000-019 138 138 138 227 227 227 F F F
154312024-02-15T03:04:24.141Z 1 020-039 139 139 139 230 230 230 F F F
154322024-02-15T03:04:24.141Z 2 040-059 139 139 139 230 230 230 F F F
154332024-02-15T03:04:24.141Z 3 060-079 138 138 138 229 229 229 F F F
154342024-02-15T03:04:24.142Z 4 080-099 134 134 134 221 221 221 F F F
154352024-02-15T03:04:24.142Z 5 100-119 129 129 129 214 214 214 F F F
154362024-02-15T03:04:24.142Z 6 120-139 133 133 133 220 220 220 F F F
154372024-02-15T03:04:24.142Z 7 140-159 134 134 134 223 223 223 F F F
154382024-02-15T03:04:24.142Z 8 160-179 130 130 130 215 215 215 F F F
154392024-02-15T03:04:24.142Z 9 180-199 138 138 138 227 227 227 F F F
154402024-02-15T03:04:24.142Z 10 200-219 138 138 138 228 228 228 F F F
154412024-02-15T03:04:24.142Z 11 220-239 138 138 138 229 229 229 F F F
154422024-02-15T03:04:24.142Z 12 240-259 136 136 136 225 225 225 F F F
154432024-02-15T03:04:24.142Z 13 260-279 138 138 138 229 229 229 F F F
154442024-02-15T03:04:24.142Z 14 280-299 138 138 138 229 229 229 F F F
154452024-02-15T03:04:24.142Z 15 300-319 136 136 136 225 225 225 F F F
154462024-02-15T03:04:24.142Z 16 320-339 136 136 136 225 225 225 F F F
154472024-02-15T03:04:24.142Z 17 340-359 138 138 138 229 229 229 F F F
154482024-02-15T03:04:24.142Z 18 360-379 138 138 138 229 229 229 F F F
154492024-02-15T03:04:24.142Z 19 380-399 138 138 138 229 229 229 F F F
154502024-02-15T03:04:24.142Z 20 400-419 134 134 134 222 222 222 F F F
154512024-02-15T03:04:24.142Z 21 420-439 139 139 139 230 230 230 F F F
154522024-02-15T03:04:24.142Z 22 440-459 138 138 138 229 229 229 F F F
154532024-02-15T03:04:24.142Z 23 460-479 132 132 132 219 219 219 F F F
154542024-02-15T03:04:24.142Z 24 480-499 135 135 135 224 224 224 F F F
154552024-02-15T03:04:24.142Z 25 500-519 138 138 138 226 226 226 F F F
154562024-02-15T03:04:24.142Z 26 520-539 136 136 136 225 225 225 F F F
154572024-02-15T03:04:24.143Z 27 540-559 136 136 136 225 225 225 F F F
154582024-02-15T03:04:24.143Z 28 560-579 136 136 136 225 225 225 F F F
154592024-02-15T03:04:24.143Z 29 580-599 136 136 136 225 225 225 F F F
154602024-02-15T03:04:24.143ZMax gen: 139, Max flush: 230
154612024-02-15T03:04:24.146Z
154622024-02-15T03:04:24.146ZBegin loop 70
154632024-02-15T03:04:24.146ZDownstairs to restart: 1
154642024-02-15T03:04:24.163Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.060783331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698}
154652024-02-15T03:04:24.163Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061082924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698}
154662024-02-15T03:04:24.168Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061130438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698}
154672024-02-15T03:04:24.169Z{"msg":"Crucible 44eb2161-df76-4ed8-afe6-00e62d28907f has session id: aab79927-f4a8-4dd8-80bd-860912e98b79","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061162304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154682024-02-15T03:04:24.169Z{"msg":"Upstairs opts: Upstairs UUID: 44eb2161-df76-4ed8-afe6-00e62d28907f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
154692024-02-15T03:04:24.169Z2024-02-15T03:04:23.061223176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154702024-02-15T03:04:24.169Z{"msg":"Crucible stats registered with UUID: 44eb2161-df76-4ed8-afe6-00e62d28907f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.06127233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154712024-02-15T03:04:24.169Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061370728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698}
154722024-02-15T03:04:24.169Z{"msg":"44eb2161-df76-4ed8-afe6-00e62d28907f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061420622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154732024-02-15T03:04:24.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061459037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154742024-02-15T03:04:24.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061484314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154752024-02-15T03:04:24.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061534327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154762024-02-15T03:04:24.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061563514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154772024-02-15T03:04:24.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061599759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154782024-02-15T03:04:24.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061618997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154792024-02-15T03:04:24.169Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061649543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154802024-02-15T03:04:24.169Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061694037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154812024-02-15T03:04:24.169Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061720964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154822024-02-15T03:04:24.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061766488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154832024-02-15T03:04:24.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.061793215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154842024-02-15T03:04:24.169Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:23.061822551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154852024-02-15T03:04:24.169Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:23.061852417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154862024-02-15T03:04:24.169Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:23.061965743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154872024-02-15T03:04:24.169Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062000289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154882024-02-15T03:04:24.169Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062025546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154892024-02-15T03:04:24.170Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062147831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154902024-02-15T03:04:24.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062367703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154912024-02-15T03:04:24.170Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062402748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154922024-02-15T03:04:24.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062534652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154932024-02-15T03:04:24.170Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062568028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154942024-02-15T03:04:24.170Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.062919704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154952024-02-15T03:04:24.170Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.063094972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154962024-02-15T03:04:24.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.06342613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154972024-02-15T03:04:24.170Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.063473534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154982024-02-15T03:04:24.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.06350397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
154992024-02-15T03:04:24.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.063565613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155002024-02-15T03:04:24.170Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.063599129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155012024-02-15T03:04:24.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:23.063624805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155022024-02-15T03:04:25.167Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.063334535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155032024-02-15T03:04:25.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.063499374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155042024-02-15T03:04:25.173Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.064426868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155052024-02-15T03:04:25.173Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.064474052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155062024-02-15T03:04:25.173Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.064944923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155072024-02-15T03:04:25.173Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065515461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155082024-02-15T03:04:25.173Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065556636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155092024-02-15T03:04:25.173Z{"msg":"[0]R flush_numbers[0..12]: [227, 230, 230, 229, 221, 214, 220, 223, 215, 227, 228, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065581763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155102024-02-15T03:04:25.173Z{"msg":"[0]R generation[0..12]: [138, 139, 139, 138, 134, 129, 133, 134, 130, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065620338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155112024-02-15T03:04:25.173Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065644365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155122024-02-15T03:04:25.173Z{"msg":"[1]R flush_numbers[0..12]: [227, 230, 230, 229, 221, 214, 220, 223, 215, 227, 228, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065670452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155132024-02-15T03:04:25.173Z{"msg":"[1]R generation[0..12]: [138, 139, 139, 138, 134, 129, 133, 134, 130, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.06568557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155142024-02-15T03:04:25.173Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065701798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155152024-02-15T03:04:25.173Z{"msg":"[2]R flush_numbers[0..12]: [227, 230, 230, 229, 221, 214, 220, 223, 215, 227, 228, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065724675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155162024-02-15T03:04:25.173Z{"msg":"[2]R generation[0..12]: [138, 139, 139, 138, 134, 129, 133, 134, 130, 138, 138, 138]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
155172024-02-15T03:04:25.173Z,"level":30,"time":"2024-02-15T03:04:24.065742593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155182024-02-15T03:04:25.173Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.06576483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155192024-02-15T03:04:25.173Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065779059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155202024-02-15T03:04:25.173Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065800746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155212024-02-15T03:04:25.173Z{"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065823353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155222024-02-15T03:04:25.173Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065855879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155232024-02-15T03:04:25.174Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065874136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155242024-02-15T03:04:25.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065906052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155252024-02-15T03:04:25.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065938528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155262024-02-15T03:04:25.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.065972044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155272024-02-15T03:04:25.174Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.06600579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155282024-02-15T03:04:25.174Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.066028717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155292024-02-15T03:04:25.174Z{"msg":"44eb2161-df76-4ed8-afe6-00e62d28907f is now active with session: aab79927-f4a8-4dd8-80bd-860912e98b79","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155302024-02-15T03:04:25.174Z,"time":"2024-02-15T03:04:24.066061133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155312024-02-15T03:04:25.174Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.06608408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698,"session_id":"aab79927-f4a8-4dd8-80bd-860912e98b79"}
155322024-02-15T03:04:25.174Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:24.066114197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698}
155332024-02-15T03:04:25.174ZLoading write count information from file "/tmp/test_repair_verify.data"
155342024-02-15T03:04:25.174ZRun Repair workload
155352024-02-15T03:04:25.174Z01/30 Write block 369 len 28672 data: 9 9 10 9 10 9 9
155362024-02-15T03:04:25.174Z02/30 Read block 335 len 4096
155372024-02-15T03:04:25.178Z03/30 Read block 79 len 24576
155382024-02-15T03:04:25.178Z04/30 Flush
155392024-02-15T03:04:25.178Z05/30 Write block 154 len 16384 data: 9 9 14 14
155402024-02-15T03:04:25.178Z06/30 Write block 54 len 4096 data: 10
155412024-02-15T03:04:25.179Z07/30 Read block 369 len 20480
155422024-02-15T03:04:25.179Z08/30 Read block 238 len 40960
155432024-02-15T03:04:25.182Z09/30 Flush
155442024-02-15T03:04:25.183Z10/30 Write block 67 len 24576 data: 10 9 9 9 10 9
155452024-02-15T03:04:25.183Z11/30 Write block 128 len 4096 data: 8
155462024-02-15T03:04:25.183Z12/30 Write block 72 len 28672 data: 10 6 6 9 9 7 6
155472024-02-15T03:04:25.198Z13/30 Read block 150 len 12288
155482024-02-15T03:04:25.198Z14/30 Read block 432 len 16384
155492024-02-15T03:04:25.198Z15/30 Read block 221 len 4096
155502024-02-15T03:04:25.198Z16/30 Read block 475 len 28672
155512024-02-15T03:04:25.198Z17/30 Write block 350 len 20480 data: 13 15 14 14 13
155522024-02-15T03:04:25.198Z18/30 Read block 6 len 36864
155532024-02-15T03:04:25.198Z19/30 Write block 397 len 40960 data: 10 10 12 10 9 8 7 7 12 12
155542024-02-15T03:04:25.198Z20/30 Read block 380 len 36864
155552024-02-15T03:04:25.204Z21/30 Read block 371 len 40960
155562024-02-15T03:04:25.204Z22/30 Flush
155572024-02-15T03:04:25.204Z23/30 Flush
155582024-02-15T03:04:25.205Z24/30 Write block 516 len 4096 data: 11
155592024-02-15T03:04:25.205Z25/30 Write block 252 len 36864 data: 11 10 10 9 10 12 12 14 11
155602024-02-15T03:04:25.205Z26/30 Read block 277 len 40960
155612024-02-15T03:04:25.205Z27/30 Write block 64 len 4096 data: 13
155622024-02-15T03:04:25.205Z28/30 Write block 250 len 40960 data: 10 10 12 11 11 10 11 13 13 15
155632024-02-15T03:04:25.205Z29/30 Read block 207 len 20480
155642024-02-15T03:04:25.205Z30/30 Read block 89 len 28672
155652024-02-15T03:04:25.205Z----------------------------------------------------------------
155662024-02-15T03:04:25.205Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
155672024-02-15T03:04:25.205ZStates: Active Active Active
155682024-02-15T03:04:25.205ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155692024-02-15T03:04:25.205Z 1 Acked 1000 Write 7 Done Done Done false
155702024-02-15T03:04:25.205Z 2 Acked 1001 Read 1 Done Done Done false
155712024-02-15T03:04:25.206Z 3 Acked 1002 Read 6 Done Done Done false
155722024-02-15T03:04:25.206Z 4 Acked 1003 Flush 0 Done Sent Done false
155732024-02-15T03:04:25.206Z 5 Acked 1004 Write 4 Done Sent Done false
155742024-02-15T03:04:25.206Z 6 Acked 1005 Write 1 Done Sent Done false
155752024-02-15T03:04:25.206Z 7 Acked 1006 Read 5 Done Sent Done false
155762024-02-15T03:04:25.206Z 8 Acked 1007 Read 10 Done Sent Done false
155772024-02-15T03:04:25.206Z 9 Acked 1008 Flush 0 Done Sent Done false
155782024-02-15T03:04:25.206Z 10 Acked 1009 Write 6 Done Sent Done false
155792024-02-15T03:04:25.207Z 11 Acked 1010 Write 1 Done Sent Done false
155802024-02-15T03:04:25.207Z 12 Acked 1011 Write 7 Done Sent Done false
155812024-02-15T03:04:25.207Z 13 Acked 1012 Read 3 Done Sent Done false
155822024-02-15T03:04:25.207Z 14 Acked 1013 Read 4 Done Sent Done false
155832024-02-15T03:04:25.207Z 15 Acked 1014 Read 1 Done Sent Done false
155842024-02-15T03:04:25.208Z 16 Acked 1015 Read 7 Done Sent Done false
155852024-02-15T03:04:25.208Z 17 Acked 1016 Write 5 Done Sent Done false
155862024-02-15T03:04:25.208Z 18 Acked 1017 Read 9 Done Sent Done false
155872024-02-15T03:04:25.208Z 19 Acked 1018 Write 10 Done Sent Done false
155882024-02-15T03:04:25.208Z 20 Acked 1019 Read 9 Done Sent Done false
155892024-02-15T03:04:25.208Z 21 Acked 1020 Read 10 Done Sent Done false
155902024-02-15T03:04:25.208Z 22 Acked 1021 Flush 0 Done Sent Done false
155912024-02-15T03:04:25.208Z 23 Acked 1022 Flush 0 Done Sent Done false
155922024-02-15T03:04:25.208Z 24 Acked 1023 Write 1 Done Sent Done false
155932024-02-15T03:04:25.208Z 25 Acked 1024 Write 9 Done Sent Done false
155942024-02-15T03:04:25.208Z 26 Acked 1025 Read 10 Done Sent Done false
155952024-02-15T03:04:25.208Z 27 Acked 1026 Write 1 Done Sent Done false
155962024-02-15T03:04:25.209Z 28 Acked 1027 Write 10 Done Sent Done false
155972024-02-15T03:04:25.209Z 29 Acked 1028 Read 5 Done Sent Done false
155982024-02-15T03:04:25.209Z 30 Acked 1029 Read 7 Done Sent Done false
155992024-02-15T03:04:25.209Z STATES DS:0 DS:1 DS:2 TOTAL
156002024-02-15T03:04:25.209Z New 0 0 0 0
156012024-02-15T03:04:25.209Z Sent 0 27 0 27
156022024-02-15T03:04:25.210Z Done 30 3 30 63
156032024-02-15T03:04:25.210Z Skipped 0 0 0 0
156042024-02-15T03:04:25.210Z Error 0 0 0 0
156052024-02-15T03:04:25.210ZLast Flush: 1022 0 1022
156062024-02-15T03:04:25.210ZDownstairs last five completed:
156072024-02-15T03:04:25.210ZUpstairs last five completed: 30 29 28 27 26
156082024-02-15T03:04:25.210Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:24.099372709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1698}
156092024-02-15T03:04:25.210ZWrote out file "/tmp/test_repair_verify.data"
156102024-02-15T03:04:25.217Z
156112024-02-15T03:04:25.218ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
156122024-02-15T03:04:25.229ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156132024-02-15T03:04:25.230Z 0 000-019 138 138 138 227 227 227 F F F
156142024-02-15T03:04:25.230Z 1 020-039 139 139 139 230 230 230 F F F
156152024-02-15T03:04:25.230Z 2 040-059 140 139 140 232 230 232 F F F <---
156162024-02-15T03:04:25.230Z 3 060-079 140 138 140 233 229 233 T F T <---
156172024-02-15T03:04:25.230Z 4 080-099 134 134 134 221 221 221 F F F
156182024-02-15T03:04:25.231Z 5 100-119 129 129 129 214 214 214 F F F
156192024-02-15T03:04:25.231Z 6 120-139 140 133 140 233 220 233 F F F <---
156202024-02-15T03:04:25.231Z 7 140-159 140 134 140 232 223 232 F F F <---
156212024-02-15T03:04:25.231Z 8 160-179 130 130 130 215 215 215 F F F
156222024-02-15T03:04:25.232Z 9 180-199 138 138 138 227 227 227 F F F
156232024-02-15T03:04:25.232Z 10 200-219 138 138 138 228 228 228 F F F
156242024-02-15T03:04:25.232Z 11 220-239 138 138 138 229 229 229 F F F
156252024-02-15T03:04:25.232Z 12 240-259 136 136 136 225 225 225 T F T <---
156262024-02-15T03:04:25.232Z 13 260-279 138 138 138 229 229 229 T F T <---
156272024-02-15T03:04:25.232Z 14 280-299 138 138 138 229 229 229 F F F
156282024-02-15T03:04:25.252Z 15 300-319 136 136 136 225 225 225 F F F
156292024-02-15T03:04:25.253Z 16 320-339 136 136 136 225 225 225 F F F
156302024-02-15T03:04:25.253Z 17 340-359 140 138 140 233 229 233 F F F <---
156312024-02-15T03:04:25.253Z 18 360-379 140 138 140 231 229 231 F T F <---
156322024-02-15T03:04:25.253Z 19 380-399 140 138 140 233 229 233 F F F <---
156332024-02-15T03:04:25.253Z 20 400-419 140 134 140 233 222 233 F F F <---
156342024-02-15T03:04:25.254Z 21 420-439 139 139 139 230 230 230 F F F
156352024-02-15T03:04:25.254Z 22 440-459 138 138 138 229 229 229 F F F
156362024-02-15T03:04:25.254Z 23 460-479 132 132 132 219 219 219 F F F
156372024-02-15T03:04:25.254Z 24 480-499 135 135 135 224 224 224 F F F
156382024-02-15T03:04:25.254Z 25 500-519 138 138 138 226 226 226 T F T <---
156392024-02-15T03:04:25.254Z 26 520-539 136 136 136 225 225 225 F F F
156402024-02-15T03:04:25.255Z 27 540-559 136 136 136 225 225 225 F F F
156412024-02-15T03:04:25.255Z 28 560-579 136 136 136 225 225 225 F F F
156422024-02-15T03:04:25.255Z 29 580-599 136 136 136 225 225 225 F F F
156432024-02-15T03:04:25.255ZMax gen: 140, Max flush: 233
156442024-02-15T03:04:25.255ZError: Difference in extent metadata found!
156452024-02-15T03:04:25.255ZA Difference in extent metadata is expected here
156462024-02-15T03:04:25.256ZOn loop 70
156472024-02-15T03:04:25.256Z
156482024-02-15T03:04:25.256ZVerifying data now
156492024-02-15T03:04:30.820ZNo change after verify
156502024-02-15T03:04:30.820ZLoop: 70 Downstairs dump after verify (and repair):
156512024-02-15T03:04:30.838ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156522024-02-15T03:04:30.838Z 0 000-019 138 138 138 227 227 227 F F F
156532024-02-15T03:04:30.838Z 1 020-039 139 139 139 230 230 230 F F F
156542024-02-15T03:04:30.838Z 2 040-059 140 140 140 232 232 232 F F F
156552024-02-15T03:04:30.838Z 3 060-079 141 141 141 234 234 234 F F F
156562024-02-15T03:04:30.838Z 4 080-099 134 134 134 221 221 221 F F F
156572024-02-15T03:04:30.838Z 5 100-119 129 129 129 214 214 214 F F F
156582024-02-15T03:04:30.838Z 6 120-139 140 140 140 233 233 233 F F F
156592024-02-15T03:04:30.838Z 7 140-159 140 140 140 232 232 232 F F F
156602024-02-15T03:04:30.838Z 8 160-179 130 130 130 215 215 215 F F F
156612024-02-15T03:04:30.841Z 9 180-199 138 138 138 227 227 227 F F F
156622024-02-15T03:04:30.841Z 10 200-219 138 138 138 228 228 228 F F F
156632024-02-15T03:04:30.841Z 11 220-239 138 138 138 229 229 229 F F F
156642024-02-15T03:04:30.841Z 12 240-259 141 141 141 234 234 234 F F F
156652024-02-15T03:04:30.842Z 13 260-279 141 141 141 234 234 234 F F F
156662024-02-15T03:04:30.842Z 14 280-299 138 138 138 229 229 229 F F F
156672024-02-15T03:04:30.842Z 15 300-319 136 136 136 225 225 225 F F F
156682024-02-15T03:04:30.842Z 16 320-339 136 136 136 225 225 225 F F F
156692024-02-15T03:04:30.842Z 17 340-359 140 140 140 233 233 233 F F F
156702024-02-15T03:04:30.842Z 18 360-379 140 140 140 231 231 231 F F F
156712024-02-15T03:04:30.842Z 19 380-399 140 140 140 233 233 233 F F F
156722024-02-15T03:04:30.842Z 20 400-419 140 140 140 233 233 233 F F F
156732024-02-15T03:04:30.842Z 21 420-439 139 139 139 230 230 230 F F F
156742024-02-15T03:04:30.842Z 22 440-459 138 138 138 229 229 229 F F F
156752024-02-15T03:04:30.842Z 23 460-479 132 132 132 219 219 219 F F F
156762024-02-15T03:04:30.842Z 24 480-499 135 135 135 224 224 224 F F F
156772024-02-15T03:04:30.842Z 25 500-519 141 141 141 234 234 234 F F F
156782024-02-15T03:04:30.842Z 26 520-539 136 136 136 225 225 225 F F F
156792024-02-15T03:04:30.842Z 27 540-559 136 136 136 225 225 225 F F F
156802024-02-15T03:04:30.842Z 28 560-579 136 136 136 225 225 225 F F F
156812024-02-15T03:04:30.842Z 29 580-599 136 136 136 225 225 225 F F F
156822024-02-15T03:04:30.842ZMax gen: 141, Max flush: 234
156832024-02-15T03:04:30.842Z
156842024-02-15T03:04:30.842ZBegin loop 71
156852024-02-15T03:04:30.842ZDownstairs to restart: 0
156862024-02-15T03:04:30.855Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.750885592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706}
156872024-02-15T03:04:30.855ZCrucible runtime is spawned
156882024-02-15T03:04:30.860Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751277972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706}
156892024-02-15T03:04:30.860Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751327726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706}
156902024-02-15T03:04:30.860Z{"msg":"Crucible dfd8ac71-5f87-4cab-983f-3d9169b0dada has session id: a4ef8160-13e0-44d2-9820-f3697be6f592","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751356882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156912024-02-15T03:04:30.860Z{"msg":"Upstairs opts: Upstairs UUID: dfd8ac71-5f87-4cab-983f-3d9169b0dada, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751421634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156922024-02-15T03:04:30.860Z{"msg":"Crucible stats registered with UUID: dfd8ac71-5f87-4cab-983f-3d9169b0dada","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751467809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156932024-02-15T03:04:30.860Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751498805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706}
156942024-02-15T03:04:30.860Z{"msg":"dfd8ac71-5f87-4cab-983f-3d9169b0dada active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751554378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156952024-02-15T03:04:30.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751624889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156962024-02-15T03:04:30.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751676262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156972024-02-15T03:04:30.861Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751719197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156982024-02-15T03:04:30.861Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751763522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
156992024-02-15T03:04:30.861Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751794028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157002024-02-15T03:04:30.861Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751835073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157012024-02-15T03:04:30.861Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751875908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157022024-02-15T03:04:30.861Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.751907354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157032024-02-15T03:04:30.861Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.75193464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157042024-02-15T03:04:30.861Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:29.751990783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157052024-02-15T03:04:30.861Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:29.752038727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157062024-02-15T03:04:30.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752080102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157072024-02-15T03:04:30.861Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752108768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157082024-02-15T03:04:30.861Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:29.752150243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157092024-02-15T03:04:30.861Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752187729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157102024-02-15T03:04:30.861Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752226904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157112024-02-15T03:04:30.861Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752265359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157122024-02-15T03:04:30.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752549583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157132024-02-15T03:04:30.861Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752581709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157142024-02-15T03:04:30.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752607896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157152024-02-15T03:04:30.861Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.752648171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157162024-02-15T03:04:30.861Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.753170105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157172024-02-15T03:04:30.861Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.753289671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157182024-02-15T03:04:30.861Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.753705198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157192024-02-15T03:04:30.861Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.753746833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157202024-02-15T03:04:30.861Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.75377287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157212024-02-15T03:04:30.862Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.753804866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157222024-02-15T03:04:30.862Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.753829493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157232024-02-15T03:04:30.862Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:29.753857229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157242024-02-15T03:04:31.864Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.753453513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157252024-02-15T03:04:31.864Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.753535812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157262024-02-15T03:04:31.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.754506981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157272024-02-15T03:04:31.872Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.754543136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157282024-02-15T03:04:31.872Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755102886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157292024-02-15T03:04:31.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755811357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157302024-02-15T03:04:31.873Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755848053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157312024-02-15T03:04:31.873Z{"msg":"[0]R flush_numbers[0..12]: [227, 230, 232, 234, 221, 214, 233, 232, 215, 227, 228, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.75586942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157322024-02-15T03:04:31.873Z{"msg":"[0]R generation[0..12]: [138, 139, 140, 141, 134, 129, 140, 140, 130, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755889697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157332024-02-15T03:04:31.873Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755917444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157342024-02-15T03:04:31.873Z{"msg":"[1]R flush_numbers[0..12]: [227, 230, 232, 234, 221, 214, 233, 232, 215, 227, 228, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755935852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157352024-02-15T03:04:31.873Z{"msg":"[1]R generation[0..12]: [138, 139, 140, 141, 134, 129, 140, 140, 130, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755961719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157362024-02-15T03:04:31.873Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.755979716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157372024-02-15T03:04:31.873Z{"msg":"[2]R flush_numbers[0..12]: [227, 230, 232, 234, 221, 214, 233, 232, 215, 227, 228, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756014212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157382024-02-15T03:04:31.873Z{"msg":"[2]R generation[0..12]: [138, 139, 140, 141, 134, 129, 140, 140, 130, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756046448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO
157392024-02-15T03:04:31.873Z"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157402024-02-15T03:04:31.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756082423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157412024-02-15T03:04:31.873Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.75610774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157422024-02-15T03:04:31.873Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756126318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157432024-02-15T03:04:31.873Z{"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756151485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157442024-02-15T03:04:31.873Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756168713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157452024-02-15T03:04:31.873Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756193929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157462024-02-15T03:04:31.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756209437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157472024-02-15T03:04:31.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756234914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157482024-02-15T03:04:31.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756252412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157492024-02-15T03:04:31.874Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756277169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157502024-02-15T03:04:31.874Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756292997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157512024-02-15T03:04:31.874Z{"msg":"dfd8ac71-5f87-4cab-983f-3d9169b0dada is now active with session: a4ef8160-13e0-44d2-9820-f3697be6f592","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756317354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157522024-02-15T03:04:31.874Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756332742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706,"session_id":"a4ef8160-13e0-44d2-9820-f3697be6f592"}
157532024-02-15T03:04:31.874Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:30.756356129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706}
157542024-02-15T03:04:31.874ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157552024-02-15T03:04:31.874ZLoading write count information from file "/tmp/test_repair_verify.data"
157562024-02-15T03:04:31.874ZRun Repair workload
157572024-02-15T03:04:31.874Z01/30 Write block 391 len 40960 data: 12 11 10 11 11 11 11 11 13 11
157582024-02-15T03:04:31.874Z02/30 Read block 312 len 40960
157592024-02-15T03:04:31.874Z03/30 Read block 376 len 36864
157602024-02-15T03:04:31.874Z04/30 Read block 256 len 12288
157612024-02-15T03:04:31.875Z05/30 Read block 96 len 40960
157622024-02-15T03:04:31.875Z06/30 Read block 407 len 24576
157632024-02-15T03:04:31.875Z07/30 Write block 68 len 16384 data: 10 10 10 11
157642024-02-15T03:04:31.875Z08/30 Read block 12 len 20480
157652024-02-15T03:04:31.879Z09/30 Read block 553 len 8192
157662024-02-15T03:04:31.879Z10/30 Write block 557 len 16384 data: 13 14 16 16
157672024-02-15T03:04:31.879Z11/30 Read block 365 len 32768
157682024-02-15T03:04:31.879Z12/30 Read block 412 len 4096
157692024-02-15T03:04:31.880Z13/30 Read block 416 len 24576
157702024-02-15T03:04:31.880Z14/30 Write block 359 len 8192 data: 15 12
157712024-02-15T03:04:31.880Z15/30 Write block 479 len 36864 data: 14 13 13 11 6 8 6 8 10
157722024-02-15T03:04:31.880Z16/30 Write block 490 len 12288 data: 13 12 14
157732024-02-15T03:04:31.880Z17/30 Read block 316 len 12288
157742024-02-15T03:04:31.880Z18/30 Write block 260 len 8192 data: 12 9
157752024-02-15T03:04:31.880Z19/30 Read block 9 len 20480
157762024-02-15T03:04:31.880Z20/30 Write block 207 len 32768 data: 8 10 9 7 8 10 10 12
157772024-02-15T03:04:31.880Z21/30 Read block 100 len 12288
157782024-02-15T03:04:31.884Z22/30 Read block 287 len 40960
157792024-02-15T03:04:31.884Z23/30 Read block 545 len 8192
157802024-02-15T03:04:31.884Z24/30 Read block 298 len 4096
157812024-02-15T03:04:31.884Z25/30 Read block 155 len 28672
157822024-02-15T03:04:31.884Z26/30 Read block 482 len 28672
157832024-02-15T03:04:31.889Z27/30 Read block 134 len 4096
157842024-02-15T03:04:31.890Z28/30 Write block 412 len 20480 data: 17 19 15 12 11
157852024-02-15T03:04:31.890Z29/30 Read block 551 len 20480
157862024-02-15T03:04:31.890Z30/30 Write block 143 len 36864 data: 11 14 13 12 12 9 9 9 9
157872024-02-15T03:04:31.890Z----------------------------------------------------------------
157882024-02-15T03:04:31.890Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
157892024-02-15T03:04:31.891ZStates: Active Active Active
157902024-02-15T03:04:31.891ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157912024-02-15T03:04:31.891Z 1 Acked 1000 Write 10 Done Done Done false
157922024-02-15T03:04:31.891Z 2 Acked 1001 Read 10 Done Done Done false
157932024-02-15T03:04:31.891Z 3 Acked 1002 Read 9 Sent Done Done false
157942024-02-15T03:04:31.891Z 4 Acked 1003 Read 3 Sent Done Done false
157952024-02-15T03:04:31.891Z 5 Acked 1004 Read 10 Sent Done Done false
157962024-02-15T03:04:31.891Z 6 Acked 1005 Read 6 Sent Done Done false
157972024-02-15T03:04:31.891Z 7 Acked 1006 Write 4 Sent Done Done false
157982024-02-15T03:04:31.891Z 8 Acked 1007 Read 5 Sent Done Done false
157992024-02-15T03:04:31.892Z 9 Acked 1008 Read 2 Sent Done Done false
158002024-02-15T03:04:31.892Z 10 Acked 1009 Write 4 Sent Done Done false
158012024-02-15T03:04:31.892Z 11 Acked 1010 Read 8 Sent Done Done false
158022024-02-15T03:04:31.892Z 12 Acked 1011 Read 1 Sent Done Done false
158032024-02-15T03:04:31.893Z 13 Acked 1012 Read 6 Sent Done Done false
158042024-02-15T03:04:31.893Z 14 Acked 1013 Write 2 Sent Done Done false
158052024-02-15T03:04:31.893Z 15 Acked 1014 Write 9 Sent Done Done false
158062024-02-15T03:04:31.894Z 16 Acked 1015 Write 3 Sent Done Done false
158072024-02-15T03:04:31.894Z 17 Acked 1016 Read 3 Sent Done Done false
158082024-02-15T03:04:31.894Z 18 Acked 1017 Write 2 Sent Done Done false
158092024-02-15T03:04:31.894Z 19 Acked 1018 Read 5 Sent Done Done false
158102024-02-15T03:04:31.895Z 20 Acked 1019 Write 8 Sent Done Done false
158112024-02-15T03:04:31.895Z 21 Acked 1020 Read 3 Sent Done Done false
158122024-02-15T03:04:31.895Z 22 Acked 1021 Read 10 Sent Done Done false
158132024-02-15T03:04:31.895Z 23 Acked 1022 Read 2 Sent Done Done false
158142024-02-15T03:04:31.895Z 24 Acked 1023 Read 1 Sent Done Done false
158152024-02-15T03:04:31.895Z 25 Acked 1024 Read 7 Sent Done Done false
158162024-02-15T03:04:31.895Z 26 Acked 1025 Read 7 Sent Done Done false
158172024-02-15T03:04:31.896Z 27 Acked 1026 Read 1 Sent Done Done false
158182024-02-15T03:04:31.896Z 28 Acked 1027 Write 5 Sent Done Done false
158192024-02-15T03:04:31.896Z 29 Acked 1028 Read 5 Sent Done Done false
158202024-02-15T03:04:31.896Z 30 Acked 1029 Write 9 Sent Sent Sent false
158212024-02-15T03:04:31.896Z STATES DS:0 DS:1 DS:2 TOTAL
158222024-02-15T03:04:31.896Z New 0 0 0 0
158232024-02-15T03:04:31.897Z Sent 28 1 1 30
158242024-02-15T03:04:31.897Z Done 2 29 29 60
158252024-02-15T03:04:31.897Z Skipped 0 0 0 0
158262024-02-15T03:04:31.897Z Error 0 0 0 0
158272024-02-15T03:04:31.897ZLast Flush: 0 0 0
158282024-02-15T03:04:31.897ZDownstairs last five completed:
158292024-02-15T03:04:31.898ZUpstairs last five completed: 30 29 28 27 26
158302024-02-15T03:04:31.898Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:30.784587872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1706}
158312024-02-15T03:04:31.920ZWrote out file "/tmp/test_repair_verify.data"
158322024-02-15T03:04:31.920Z
158332024-02-15T03:04:31.920ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158342024-02-15T03:04:31.933ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158352024-02-15T03:04:31.934Z 0 000-019 138 138 138 227 227 227 F F F
158362024-02-15T03:04:31.934Z 1 020-039 139 139 139 230 230 230 F F F
158372024-02-15T03:04:31.934Z 2 040-059 140 140 140 232 232 232 F F F
158382024-02-15T03:04:31.934Z 3 060-079 141 141 141 234 234 234 F T T <---
158392024-02-15T03:04:31.934Z 4 080-099 134 134 134 221 221 221 F F F
158402024-02-15T03:04:31.934Z 5 100-119 129 129 129 214 214 214 F F F
158412024-02-15T03:04:31.934Z 6 120-139 140 140 140 233 233 233 F F F
158422024-02-15T03:04:31.934Z 7 140-159 140 140 140 232 232 232 F T T <---
158432024-02-15T03:04:31.934Z 8 160-179 130 130 130 215 215 215 F F F
158442024-02-15T03:04:31.934Z 9 180-199 138 138 138 227 227 227 F F F
158452024-02-15T03:04:31.935Z 10 200-219 138 138 138 228 228 228 F T T <---
158462024-02-15T03:04:31.935Z 11 220-239 138 138 138 229 229 229 F F F
158472024-02-15T03:04:31.935Z 12 240-259 141 141 141 234 234 234 F F F
158482024-02-15T03:04:31.935Z 13 260-279 141 141 141 234 234 234 F T T <---
158492024-02-15T03:04:31.935Z 14 280-299 138 138 138 229 229 229 F F F
158502024-02-15T03:04:31.935Z 15 300-319 136 136 136 225 225 225 F F F
158512024-02-15T03:04:31.935Z 16 320-339 136 136 136 225 225 225 F F F
158522024-02-15T03:04:31.935Z 17 340-359 140 140 140 233 233 233 F T T <---
158532024-02-15T03:04:31.935Z 18 360-379 140 140 140 231 231 231 F T T <---
158542024-02-15T03:04:31.935Z 19 380-399 140 140 140 233 233 233 T T T
158552024-02-15T03:04:31.935Z 20 400-419 140 140 140 233 233 233 T T T
158562024-02-15T03:04:31.935Z 21 420-439 139 139 139 230 230 230 F F F
158572024-02-15T03:04:31.936Z 22 440-459 138 138 138 229 229 229 F F F
158582024-02-15T03:04:31.936Z 23 460-479 132 132 132 219 219 219 F T T <---
158592024-02-15T03:04:31.936Z 24 480-499 135 135 135 224 224 224 F T T <---
158602024-02-15T03:04:31.936Z 25 500-519 141 141 141 234 234 234 F F F
158612024-02-15T03:04:31.936Z 26 520-539 136 136 136 225 225 225 F F F
158622024-02-15T03:04:31.936Z 27 540-559 136 136 136 225 225 225 F T T <---
158632024-02-15T03:04:31.936Z 28 560-579 136 136 136 225 225 225 F T T <---
158642024-02-15T03:04:31.936Z 29 580-599 136 136 136 225 225 225 F F F
158652024-02-15T03:04:31.936ZMax gen: 141, Max flush: 234
158662024-02-15T03:04:31.936ZError: Difference in extent metadata found!
158672024-02-15T03:04:31.941ZA Difference in extent metadata is expected here
158682024-02-15T03:04:31.941ZOn loop 71
158692024-02-15T03:04:31.941Z
158702024-02-15T03:04:31.941ZVerifying data now
158712024-02-15T03:04:37.597ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
158722024-02-15T03:04:37.604Z415,417c415,418
158732024-02-15T03:04:37.604Z< 16,
158742024-02-15T03:04:37.604Z< 18,
158752024-02-15T03:04:37.604Z< 14,
158762024-02-15T03:04:37.604Z---
158772024-02-15T03:04:37.604Z> 17,
158782024-02-15T03:04:37.604Z> 19,
158792024-02-15T03:04:37.604Z> 15,
158802024-02-15T03:04:37.604Z> 12,
158812024-02-15T03:04:37.604Z419d419
158822024-02-15T03:04:37.604Z< 10,
158832024-02-15T03:04:37.605Zdiff found after verify
158842024-02-15T03:04:37.612ZLoop: 71 Downstairs dump after verify (and repair):
158852024-02-15T03:04:37.621ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158862024-02-15T03:04:37.621Z 0 000-019 138 138 138 227 227 227 F F F
158872024-02-15T03:04:37.621Z 1 020-039 139 139 139 230 230 230 F F F
158882024-02-15T03:04:37.621Z 2 040-059 140 140 140 232 232 232 F F F
158892024-02-15T03:04:37.621Z 3 060-079 142 142 142 235 235 235 F F F
158902024-02-15T03:04:37.622Z 4 080-099 134 134 134 221 221 221 F F F
158912024-02-15T03:04:37.622Z 5 100-119 129 129 129 214 214 214 F F F
158922024-02-15T03:04:37.622Z 6 120-139 140 140 140 233 233 233 F F F
158932024-02-15T03:04:37.622Z 7 140-159 142 142 142 235 235 235 F F F
158942024-02-15T03:04:37.622Z 8 160-179 130 130 130 215 215 215 F F F
158952024-02-15T03:04:37.628Z 9 180-199 138 138 138 227 227 227 F F F
158962024-02-15T03:04:37.628Z 10 200-219 142 142 142 235 235 235 F F F
158972024-02-15T03:04:37.628Z 11 220-239 138 138 138 229 229 229 F F F
158982024-02-15T03:04:37.628Z 12 240-259 141 141 141 234 234 234 F F F
158992024-02-15T03:04:37.628Z 13 260-279 142 142 142 235 235 235 F F F
159002024-02-15T03:04:37.628Z 14 280-299 138 138 138 229 229 229 F F F
159012024-02-15T03:04:37.629Z 15 300-319 136 136 136 225 225 225 F F F
159022024-02-15T03:04:37.629Z 16 320-339 136 136 136 225 225 225 F F F
159032024-02-15T03:04:37.629Z 17 340-359 142 142 142 235 235 235 F F F
159042024-02-15T03:04:37.629Z 18 360-379 142 142 142 235 235 235 F F F
159052024-02-15T03:04:37.629Z 19 380-399 142 142 142 235 235 235 F F F
159062024-02-15T03:04:37.629Z 20 400-419 142 142 142 235 235 235 F F F
159072024-02-15T03:04:37.630Z 21 420-439 139 139 139 230 230 230 F F F
159082024-02-15T03:04:37.630Z 22 440-459 138 138 138 229 229 229 F F F
159092024-02-15T03:04:37.630Z 23 460-479 142 142 142 235 235 235 F F F
159102024-02-15T03:04:37.630Z 24 480-499 142 142 142 235 235 235 F F F
159112024-02-15T03:04:37.630Z 25 500-519 141 141 141 234 234 234 F F F
159122024-02-15T03:04:37.630Z 26 520-539 136 136 136 225 225 225 F F F
159132024-02-15T03:04:37.630Z 27 540-559 142 142 142 235 235 235 F F F
159142024-02-15T03:04:37.630Z 28 560-579 142 142 142 235 235 235 F F F
159152024-02-15T03:04:37.630Z 29 580-599 136 136 136 225 225 225 F F F
159162024-02-15T03:04:37.630ZMax gen: 142, Max flush: 235
159172024-02-15T03:04:37.630Z
159182024-02-15T03:04:37.630ZBegin loop 72
159192024-02-15T03:04:37.631ZDownstairs to restart: 2
159202024-02-15T03:04:37.647Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542156686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717}
159212024-02-15T03:04:37.647Z{Crucible runtime is spawned
159222024-02-15T03:04:37.647Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542481235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717}
159232024-02-15T03:04:37.654Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.54252719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717}
159242024-02-15T03:04:37.654Z{"msg":"Crucible 4ea2bf58-e941-4886-816e-cbab40399ab9 has session id: 3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542581543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159252024-02-15T03:04:37.654Z{"msg":"Upstairs opts: Upstairs UUID: 4ea2bf58-e941-4886-816e-cbab40399ab9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542639616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159262024-02-15T03:04:37.654Z{"msg":"Crucible stats registered with UUID: 4ea2bf58-e941-4886-816e-cbab40399ab9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542679701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159272024-02-15T03:04:37.654Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542710747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717}
159282024-02-15T03:04:37.654Z{"msg":"4ea2bf58-e941-4886-816e-cbab40399ab9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542750122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159292024-02-15T03:04:37.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542780548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159302024-02-15T03:04:37.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542826362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159312024-02-15T03:04:37.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542864767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159322024-02-15T03:04:37.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542895214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159332024-02-15T03:04:37.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.54292576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159342024-02-15T03:04:37.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.542957776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159352024-02-15T03:04:37.655Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.5430051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159362024-02-15T03:04:37.655Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.54316414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159372024-02-15T03:04:37.655Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543207404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159382024-02-15T03:04:37.655Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:36.54324663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159392024-02-15T03:04:37.655Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:36.543267347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159402024-02-15T03:04:37.655Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543302143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159412024-02-15T03:04:37.656Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:36.543329879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159422024-02-15T03:04:37.656Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543360885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159432024-02-15T03:04:37.656Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543386752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159442024-02-15T03:04:37.656Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543411079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159452024-02-15T03:04:37.656Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543466752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159462024-02-15T03:04:37.670Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543673516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159472024-02-15T03:04:37.670Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543706532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159482024-02-15T03:04:37.670Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.54372234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159492024-02-15T03:04:37.671Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.543736208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159502024-02-15T03:04:37.671Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.544301877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159512024-02-15T03:04:37.671Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.544339853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159522024-02-15T03:04:37.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.544901652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159532024-02-15T03:04:37.671Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.544937738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159542024-02-15T03:04:37.671Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.544952896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159552024-02-15T03:04:37.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.544966684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159562024-02-15T03:04:37.671Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.544988361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159572024-02-15T03:04:37.671Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:36.54500232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159582024-02-15T03:04:38.649Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.544682023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159592024-02-15T03:04:38.649Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.544806698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"io task","client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159602024-02-15T03:04:38.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.54606501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159612024-02-15T03:04:38.655Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.546107275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159622024-02-15T03:04:38.655Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.546609032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159632024-02-15T03:04:38.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547167372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159642024-02-15T03:04:38.655Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547222165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159652024-02-15T03:04:38.655Z{"msg":"[0]R flush_numbers[0..12]: [227, 230, 232, 235, 221, 214, 233, 235, 215, 227, 235, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547267539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159662024-02-15T03:04:38.655Z{"msg":"[0]R generation[0..12]: [138, 139, 140, 142, 134, 129, 140, 142, 130, 138, 142, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547289236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159672024-02-15T03:04:38.655Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547317233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159682024-02-15T03:04:38.655Z{"msg":"[1]R flush_numbers[0..12]: [227, 230, 232, 235, 221, 214, 233, 235, 215, 227, 235, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.54733635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159692024-02-15T03:04:38.656Z{"msg":"[1]R generation[0..12]: [138, 139, 140, 142, 134, 129, 140, 142, 130, 138, 142, 138]","Wait for a query_work_queue command to finish before sending IO
159702024-02-15T03:04:38.656Zv":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547359068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159712024-02-15T03:04:38.656Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547377575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159722024-02-15T03:04:38.656Z{"msg":"[2]R flush_numbers[0..12]: [227, 230, 232, 235, 221, 214, 233, 235, 215, 227, 235, 229]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547404922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159732024-02-15T03:04:38.656Z{"msg":"[2]R generation[0..12]: [138, 139, 140, 142, 134, 129, 140, 142, 130, 138, 142, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.54742374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159742024-02-15T03:04:38.656Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547451456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159752024-02-15T03:04:38.656Z{"msg":"Max found gen is 143","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547469114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159762024-02-15T03:04:38.656Z{"msg":"Generation requested: 144 >= found:143","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547495551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159772024-02-15T03:04:38.656Z{"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547512288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159782024-02-15T03:04:38.656Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547538065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"":"downstairs","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159792024-02-15T03:04:38.656Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547555513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159802024-02-15T03:04:38.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.54757934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"0","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159812024-02-15T03:04:38.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547597068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"1","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159822024-02-15T03:04:38.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547623425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"client":"2","":"downstairs","session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159832024-02-15T03:04:38.657Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547640552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159842024-02-15T03:04:38.657Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547665749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159852024-02-15T03:04:38.657Z{"msg":"4ea2bf58-e941-4886-816e-cbab40399ab9 is now active with session: 3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547683237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159862024-02-15T03:04:38.657Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547707004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717,"session_id":"3674fb4e-cdd8-43ab-95e3-4f0ab9c0517f"}
159872024-02-15T03:04:38.657Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:37.547723032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717}
159882024-02-15T03:04:38.657ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159892024-02-15T03:04:38.657ZLoading write count information from file "/tmp/test_repair_verify.data"
159902024-02-15T03:04:38.657ZRun Repair workload
159912024-02-15T03:04:38.657Z01/30 Write block 388 len 12288 data: 14 15 13
159922024-02-15T03:04:38.661Z02/30 Read block 142 len 36864
159932024-02-15T03:04:38.661Z03/30 Write block 372 len 8192 data: 10 11
159942024-02-15T03:04:38.661Z04/30 Write block 188 len 32768 data: 12 15 13 12 12 15 15 16
159952024-02-15T03:04:38.661Z05/30 Write block 289 len 12288 data: 13 20 18
159962024-02-15T03:04:38.661Z06/30 Read block 182 len 36864
159972024-02-15T03:04:38.661Z07/30 Read block 427 len 12288
159982024-02-15T03:04:38.661Z08/30 Write block 56 len 8192 data: 10 13
159992024-02-15T03:04:38.661Z09/30 Read block 58 len 16384
160002024-02-15T03:04:38.662Z10/30 Read block 521 len 28672
160012024-02-15T03:04:38.672Z11/30 Read block 174 len 24576
160022024-02-15T03:04:38.672Z12/30 Flush
160032024-02-15T03:04:38.672Z13/30 Flush
160042024-02-15T03:04:38.672Z14/30 Write block 152 len 40960 data: 8 10 10 10 15 15 13 10 8 9
160052024-02-15T03:04:38.675Z15/30 Read block 135 len 16384
160062024-02-15T03:04:38.675Z16/30 Write block 235 len 36864 data: 14 15 13 12 12 12 13 16 12
160072024-02-15T03:04:38.675Z17/30 Read block 162 len 4096
160082024-02-15T03:04:38.676Z18/30 Write block 347 len 16384 data: 13 11 12 14
160092024-02-15T03:04:38.676Z19/30 Read block 33 len 20480
160102024-02-15T03:04:38.676Z20/30 Write block 483 len 20480 data: 7 9 7 9 11
160112024-02-15T03:04:38.676Z21/30 Read block 54 len 16384
160122024-02-15T03:04:38.676Z22/30 Write block 137 len 36864 data: 11 12 10 12 14 13 12 15 14
160132024-02-15T03:04:38.676Z23/30 Read block 16 len 12288
160142024-02-15T03:04:38.679Z24/30 Write block 91 len 36864 data: 9 8 7 11 10 14 16 16 14
160152024-02-15T03:04:38.679Z25/30 Write block 187 len 40960 data: 12 13 16 14 13 13 16 16 17 13
160162024-02-15T03:04:38.679Z26/30 Read block 107 len 4096
160172024-02-15T03:04:38.679Z27/30 Read block 48 len 32768
160182024-02-15T03:04:38.683Z28/30 Read block 457 len 40960
160192024-02-15T03:04:38.683Z29/30 Read block 95 len 32768
160202024-02-15T03:04:38.683Z30/30 Write block 354 len 4096 data: 14
160212024-02-15T03:04:38.688Z----------------------------------------------------------------
160222024-02-15T03:04:38.688Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
160232024-02-15T03:04:38.688ZStates: Active Active Active
160242024-02-15T03:04:38.688ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
160252024-02-15T03:04:38.688Z 1 Acked 1000 Write 3 Done Done Done false
160262024-02-15T03:04:38.688Z 2 Acked 1001 Read 9 Done Done Sent false
160272024-02-15T03:04:38.688Z 3 Acked 1002 Write 2 Done Done Sent false
160282024-02-15T03:04:38.688Z 4 Acked 1003 Write 8 Done Done Sent false
160292024-02-15T03:04:38.688Z 5 Acked 1004 Write 3 Done Done Sent false
160302024-02-15T03:04:38.689Z 6 Acked 1005 Read 9 Done Done Sent false
160312024-02-15T03:04:38.689Z 7 Acked 1006 Read 3 Done Done Sent false
160322024-02-15T03:04:38.689Z 8 Acked 1007 Write 2 Done Done Sent false
160332024-02-15T03:04:38.689Z 9 Acked 1008 Read 4 Done Done Sent false
160342024-02-15T03:04:38.689Z 10 Acked 1009 Read 7 Done Done Sent false
160352024-02-15T03:04:38.689Z 11 Acked 1010 Read 6 Done Done Sent false
160362024-02-15T03:04:38.689Z 12 Acked 1011 Flush 0 Done Done Sent false
160372024-02-15T03:04:38.689Z 13 Acked 1012 Flush 0 Done Done Sent false
160382024-02-15T03:04:38.689Z 14 Acked 1013 Write 10 Done Done Sent false
160392024-02-15T03:04:38.689Z 15 Acked 1014 Read 4 Done Done Sent false
160402024-02-15T03:04:38.689Z 16 Acked 1015 Write 9 Done Done Sent false
160412024-02-15T03:04:38.689Z 17 Acked 1016 Read 1 Done Done Sent false
160422024-02-15T03:04:38.689Z 18 Acked 1017 Write 4 Done Done Sent false
160432024-02-15T03:04:38.689Z 19 Acked 1018 Read 5 Done Done Sent false
160442024-02-15T03:04:38.689Z 20 Acked 1019 Write 5 Done Done Sent false
160452024-02-15T03:04:38.689Z 21 Acked 1020 Read 4 Done Done Sent false
160462024-02-15T03:04:38.689Z 22 Acked 1021 Write 9 Done Done Sent false
160472024-02-15T03:04:38.689Z 23 Acked 1022 Read 3 Done Done Sent false
160482024-02-15T03:04:38.689Z 24 Acked 1023 Write 9 Done Done Sent false
160492024-02-15T03:04:38.690Z 25 Acked 1024 Write 10 Done Done Sent false
160502024-02-15T03:04:38.690Z 26 Acked 1025 Read 1 Done Done Sent false
160512024-02-15T03:04:38.690Z 27 Acked 1026 Read 8 Done Done Sent false
160522024-02-15T03:04:38.690Z 28 Acked 1027 Read 10 Done Done Sent false
160532024-02-15T03:04:38.690Z 29 Acked 1028 Read 8 Done Done Sent false
160542024-02-15T03:04:38.690Z 30 Acked 1029 Write 1 Sent Sent Sent false
160552024-02-15T03:04:38.690Z STATES DS:0 DS:1 DS:2 TOTAL
160562024-02-15T03:04:38.690Z New 0 0 0 0
160572024-02-15T03:04:38.690Z Sent 1 1 29 31
160582024-02-15T03:04:38.691Z Done 29 29 1 59
160592024-02-15T03:04:38.691Z Skipped 0 0 0 0
160602024-02-15T03:04:38.691Z Error 0 0 0 0
160612024-02-15T03:04:38.691ZLast Flush: 1012 1012 0
160622024-02-15T03:04:38.691ZDownstairs last five completed:
160632024-02-15T03:04:38.691ZUpstairs last five completed: 30 29 28 27 26
160642024-02-15T03:04:38.691Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:37.579389044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1717}
160652024-02-15T03:04:38.691ZWrote out file "/tmp/test_repair_verify.data"
160662024-02-15T03:04:38.696Z
160672024-02-15T03:04:38.696ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160682024-02-15T03:04:38.714ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160692024-02-15T03:04:38.714Z 0 000-019 138 138 138 227 227 227 F F F
160702024-02-15T03:04:38.714Z 1 020-039 139 139 139 230 230 230 F F F
160712024-02-15T03:04:38.714Z 2 040-059 144 144 140 236 236 232 F F F <---
160722024-02-15T03:04:38.714Z 3 060-079 142 142 142 235 235 235 F F F
160732024-02-15T03:04:38.714Z 4 080-099 134 134 134 221 221 221 T T F <---
160742024-02-15T03:04:38.714Z 5 100-119 129 129 129 214 214 214 F F F
160752024-02-15T03:04:38.714Z 6 120-139 140 140 140 233 233 233 T T F <---
160762024-02-15T03:04:38.714Z 7 140-159 142 142 142 235 235 235 T T F <---
160772024-02-15T03:04:38.714Z 8 160-179 130 130 130 215 215 215 T T F <---
160782024-02-15T03:04:38.714Z 9 180-199 144 144 138 236 236 227 T T F <---
160792024-02-15T03:04:38.714Z 10 200-219 142 142 142 235 235 235 F F F
160802024-02-15T03:04:38.723Z 11 220-239 138 138 138 229 229 229 T T F <---
160812024-02-15T03:04:38.723Z 12 240-259 141 141 141 234 234 234 T T F <---
160822024-02-15T03:04:38.723Z 13 260-279 142 142 142 235 235 235 F F F
160832024-02-15T03:04:38.723Z 14 280-299 144 144 138 236 236 229 F F F <---
160842024-02-15T03:04:38.723Z 15 300-319 136 136 136 225 225 225 F F F
160852024-02-15T03:04:38.723Z 16 320-339 136 136 136 225 225 225 F F F
160862024-02-15T03:04:38.723Z 17 340-359 142 142 142 235 235 235 T T F <---
160872024-02-15T03:04:38.724Z 18 360-379 144 144 142 236 236 235 F F F <---
160882024-02-15T03:04:38.724Z 19 380-399 144 144 142 236 236 235 F F T <---
160892024-02-15T03:04:38.724Z 20 400-419 142 142 142 235 235 235 F F F
160902024-02-15T03:04:38.724Z 21 420-439 139 139 139 230 230 230 F F F
160912024-02-15T03:04:38.724Z 22 440-459 138 138 138 229 229 229 F F F
160922024-02-15T03:04:38.724Z 23 460-479 142 142 142 235 235 235 F F F
160932024-02-15T03:04:38.725Z 24 480-499 142 142 142 235 235 235 T T F <---
160942024-02-15T03:04:38.725Z 25 500-519 141 141 141 234 234 234 F F F
160952024-02-15T03:04:38.725Z 26 520-539 136 136 136 225 225 225 F F F
160962024-02-15T03:04:38.725Z 27 540-559 142 142 142 235 235 235 F F F
160972024-02-15T03:04:38.725Z 28 560-579 142 142 142 235 235 235 F F F
160982024-02-15T03:04:38.725Z 29 580-599 136 136 136 225 225 225 F F F
160992024-02-15T03:04:38.726ZMax gen: 144, Max flush: 236
161002024-02-15T03:04:38.726ZError: Difference in extent metadata found!
161012024-02-15T03:04:38.726ZA Difference in extent metadata is expected here
161022024-02-15T03:04:38.726ZOn loop 72
161032024-02-15T03:04:38.726Z
161042024-02-15T03:04:38.727ZVerifying data now
161052024-02-15T03:04:44.393ZNo change after verify
161062024-02-15T03:04:44.394ZLoop: 72 Downstairs dump after verify (and repair):
161072024-02-15T03:04:44.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161082024-02-15T03:04:44.402Z 0 000-019 138 138 138 227 227 227 F F F
161092024-02-15T03:04:44.402Z 1 020-039 139 139 139 230 230 230 F F F
161102024-02-15T03:04:44.402Z 2 040-059 144 144 144 236 236 236 F F F
161112024-02-15T03:04:44.402Z 3 060-079 142 142 142 235 235 235 F F F
161122024-02-15T03:04:44.402Z 4 080-099 145 145 145 237 237 237 F F F
161132024-02-15T03:04:44.402Z 5 100-119 129 129 129 214 214 214 F F F
161142024-02-15T03:04:44.402Z 6 120-139 145 145 145 237 237 237 F F F
161152024-02-15T03:04:44.402Z 7 140-159 145 145 145 237 237 237 F F F
161162024-02-15T03:04:44.402Z 8 160-179 145 145 145 237 237 237 F F F
161172024-02-15T03:04:44.402Z 9 180-199 145 145 145 237 237 237 F F F
161182024-02-15T03:04:44.402Z 10 200-219 142 142 142 235 235 235 F F F
161192024-02-15T03:04:44.406Z 11 220-239 145 145 145 237 237 237 F F F
161202024-02-15T03:04:44.406Z 12 240-259 145 145 145 237 237 237 F F F
161212024-02-15T03:04:44.406Z 13 260-279 142 142 142 235 235 235 F F F
161222024-02-15T03:04:44.406Z 14 280-299 144 144 144 236 236 236 F F F
161232024-02-15T03:04:44.406Z 15 300-319 136 136 136 225 225 225 F F F
161242024-02-15T03:04:44.406Z 16 320-339 136 136 136 225 225 225 F F F
161252024-02-15T03:04:44.406Z 17 340-359 145 145 145 237 237 237 F F F
161262024-02-15T03:04:44.406Z 18 360-379 144 144 144 236 236 236 F F F
161272024-02-15T03:04:44.406Z 19 380-399 144 144 144 236 236 236 F F F
161282024-02-15T03:04:44.406Z 20 400-419 142 142 142 235 235 235 F F F
161292024-02-15T03:04:44.406Z 21 420-439 139 139 139 230 230 230 F F F
161302024-02-15T03:04:44.406Z 22 440-459 138 138 138 229 229 229 F F F
161312024-02-15T03:04:44.406Z 23 460-479 142 142 142 235 235 235 F F F
161322024-02-15T03:04:44.406Z 24 480-499 145 145 145 237 237 237 F F F
161332024-02-15T03:04:44.407Z 25 500-519 141 141 141 234 234 234 F F F
161342024-02-15T03:04:44.407Z 26 520-539 136 136 136 225 225 225 F F F
161352024-02-15T03:04:44.407Z 27 540-559 142 142 142 235 235 235 F F F
161362024-02-15T03:04:44.407Z 28 560-579 142 142 142 235 235 235 F F F
161372024-02-15T03:04:44.407Z 29 580-599 136 136 136 225 225 225 F F F
161382024-02-15T03:04:44.407ZMax gen: 145, Max flush: 237
161392024-02-15T03:04:44.407Z
161402024-02-15T03:04:44.407ZBegin loop 73
161412024-02-15T03:04:44.407ZDownstairs to restart: 1
161422024-02-15T03:04:44.426Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.320595338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725}
161432024-02-15T03:04:44.426Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.320853865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725}
161442024-02-15T03:04:44.426Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.320871553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725}
161452024-02-15T03:04:44.426Z{"msg":"Crucible d1e99ddf-44a7-4e33-a984-5bcbac8b4f84 has session id: 48454966-ad0a-487d-ba06-c740c8d2eb1a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.320886432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161462024-02-15T03:04:44.432Z{"msg":"Upstairs opts: Upstairs UUID: d1e99ddf-44a7-4e33-a984-5bcbac8b4f84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.320929116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161472024-02-15T03:04:44.433Z{"msg":"Crucible stats registered with UUID: d1e99ddf-44a7-4e33-a984-5bcbac8b4f84","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.320954143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161482024-02-15T03:04:44.433ZCrucible runtime is spawned
161492024-02-15T03:04:44.433Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321212111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725}
161502024-02-15T03:04:44.433Z{"msg":"d1e99ddf-44a7-4e33-a984-5bcbac8b4f84 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321324846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161512024-02-15T03:04:44.433Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321361322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161522024-02-15T03:04:44.433Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.32138014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161532024-02-15T03:04:44.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321415485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161542024-02-15T03:04:44.434Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321443942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161552024-02-15T03:04:44.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321477547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161562024-02-15T03:04:44.434Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321541969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161572024-02-15T03:04:44.434Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321569866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161582024-02-15T03:04:44.434Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.32161706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161592024-02-15T03:04:44.434Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321661055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161602024-02-15T03:04:44.434Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:43.32169198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161612024-02-15T03:04:44.434Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:43.321732086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161622024-02-15T03:04:44.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321763871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161632024-02-15T03:04:44.434Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:43.321794048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161642024-02-15T03:04:44.434Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321807436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161652024-02-15T03:04:44.434Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321820855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161662024-02-15T03:04:44.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.321834663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161672024-02-15T03:04:44.434Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.32209555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161682024-02-15T03:04:44.434Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.322570121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161692024-02-15T03:04:44.434Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.322616285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161702024-02-15T03:04:44.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.322666658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161712024-02-15T03:04:44.435Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.322704654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161722024-02-15T03:04:44.435Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.323244856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161732024-02-15T03:04:44.435Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.323439781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161742024-02-15T03:04:44.435Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.32377006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161752024-02-15T03:04:44.435Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.323839262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161762024-02-15T03:04:44.435Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.323876647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161772024-02-15T03:04:44.435Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.323897904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161782024-02-15T03:04:44.435Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.32392998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161792024-02-15T03:04:44.436Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:43.323945668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161802024-02-15T03:04:45.430Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.322731693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161812024-02-15T03:04:45.431Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.322894093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"io task","client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161822024-02-15T03:04:45.481Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.323805399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161832024-02-15T03:04:45.481Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.323841474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161842024-02-15T03:04:45.481Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.324333112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161852024-02-15T03:04:45.481Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.324885643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161862024-02-15T03:04:45.481Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.324920209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161872024-02-15T03:04:45.481Z{"msg":"[0]R flush_numbers[0..12]: [227, 230, 236, 235, 237, 214, 237, 237, 237, 237, 235, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.324939187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161882024-02-15T03:04:45.481Z{"msg":"[0]R generation[0..12]: [138, 139, 144, 142, 145, 129, 145, 145, 145, 145, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.324977472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161892024-02-15T03:04:45.481Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325002409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161902024-02-15T03:04:45.482Z{"msg":"[1]R flush_numbers[0..12]: [227, 230, 236, 235, 237, 214, 237, 237, 237, 237, 235, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325038934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161912024-02-15T03:04:45.482Z{"msg":"[1]R generation[0..12]: [138, 139, 144, 142, 145, 129, 145, 145, 145, 145, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325058062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161922024-02-15T03:04:45.482Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325084238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161932024-02-15T03:04:45.482Z{"msg":"[2]R flush_numbers[0..12]: [227, 230, 236, 235, 237, 214, 237, 237, 237, 237, 235, 237]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
161942024-02-15T03:04:45.482Z,"time":"2024-02-15T03:04:44.325103036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161952024-02-15T03:04:45.482Z{"msg":"[2]R generation[0..12]: [138, 139, 144, 142, 145, 129, 145, 145, 145, 145, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325134182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161962024-02-15T03:04:45.482Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.32515227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161972024-02-15T03:04:45.482Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325177467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161982024-02-15T03:04:45.482Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325194704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
161992024-02-15T03:04:45.482Z{"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325214272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162002024-02-15T03:04:45.482Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.32522814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"":"downstairs","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162012024-02-15T03:04:45.482Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325248868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162022024-02-15T03:04:45.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325269685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"0","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162032024-02-15T03:04:45.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325301741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"1","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162042024-02-15T03:04:45.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325324778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"client":"2","":"downstairs","session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162052024-02-15T03:04:45.482Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325355784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162062024-02-15T03:04:45.482Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.32538927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162072024-02-15T03:04:45.482Z{"msg":"d1e99ddf-44a7-4e33-a984-5bcbac8b4f84 is now active with session: 48454966-ad0a-487d-ba06-c740c8d2eb1a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325427375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162082024-02-15T03:04:45.482Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325464261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725,"session_id":"48454966-ad0a-487d-ba06-c740c8d2eb1a"}
162092024-02-15T03:04:45.482Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:44.325513305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725}
162102024-02-15T03:04:45.482ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
162112024-02-15T03:04:45.482ZLoading write count information from file "/tmp/test_repair_verify.data"
162122024-02-15T03:04:45.482ZRun Repair workload
162132024-02-15T03:04:45.482Z01/30 Write block 344 len 36864 data: 11 12 14 14 12 13 15 16 15
162142024-02-15T03:04:45.483Z02/30 Read block 408 len 12288
162152024-02-15T03:04:45.483Z03/30 Read block 292 len 20480
162162024-02-15T03:04:45.483Z04/30 Read block 339 len 8192
162172024-02-15T03:04:45.483Z05/30 Read block 388 len 8192
162182024-02-15T03:04:45.483Z06/30 Write block 161 len 4096 data: 10
162192024-02-15T03:04:45.483Z07/30 Read block 261 len 24576
162202024-02-15T03:04:45.483Z08/30 Read block 127 len 16384
162212024-02-15T03:04:45.483Z09/30 Write block 504 len 16384 data: 14 12 11 8
162222024-02-15T03:04:45.483Z10/30 Write block 514 len 28672 data: 13 10 12 10 9 8 6
162232024-02-15T03:04:45.483Z11/30 Read block 594 len 12288
162242024-02-15T03:04:45.483Z12/30 Write block 479 len 4096 data: 15
162252024-02-15T03:04:45.483Z13/30 Write block 409 len 24576 data: 18 19 19 17 19 15
162262024-02-15T03:04:45.483Z14/30 Write block 407 len 8192 data: 14 14
162272024-02-15T03:04:45.483Z15/30 Write block 534 len 16384 data: 9 11 8 11
162282024-02-15T03:04:45.483Z16/30 Read block 567 len 28672
162292024-02-15T03:04:45.483Z17/30 Write block 324 len 16384 data: 9 11 12 11
162302024-02-15T03:04:45.483Z18/30 Write block 22 len 28672 data: 5 3 5 5 7 8 9
162312024-02-15T03:04:45.483Z19/30 Read block 553 len 24576
162322024-02-15T03:04:45.483Z20/30 Flush
162332024-02-15T03:04:45.483Z21/30 Write block 97 len 4096 data: 17
162342024-02-15T03:04:45.483Z22/30 Flush
162352024-02-15T03:04:45.483Z23/30 Write block 286 len 32768 data: 14 15 11 14 21 19 17 15
162362024-02-15T03:04:45.483Z24/30 Read block 245 len 4096
162372024-02-15T03:04:45.483Z25/30 Write block 11 len 36864 data: 9 7 6 7 6 6 7 9 8
162382024-02-15T03:04:45.483Z26/30 Read block 592 len 8192
162392024-02-15T03:04:45.483Z27/30 Read block 301 len 12288
162402024-02-15T03:04:45.484Z28/30 Write block 109 len 12288 data: 7 7 8
162412024-02-15T03:04:45.484Z29/30 Read block 202 len 40960
162422024-02-15T03:04:45.484Z30/30 Write block 475 len 8192 data: 7 7
162432024-02-15T03:04:45.484Z----------------------------------------------------------------
162442024-02-15T03:04:45.484Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
162452024-02-15T03:04:45.484ZStates: Active Active Active
162462024-02-15T03:04:45.484ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162472024-02-15T03:04:45.484Z 1 Acked 1000 Write 9 Done Done Done false
162482024-02-15T03:04:45.484Z 2 Acked 1001 Read 3 Done Done Done false
162492024-02-15T03:04:45.484Z 3 Acked 1002 Read 5 Done Done Done false
162502024-02-15T03:04:45.484Z 4 Acked 1003 Read 2 Done Done Done false
162512024-02-15T03:04:45.484Z 5 Acked 1004 Read 2 Done Done Done false
162522024-02-15T03:04:45.484Z 6 Acked 1005 Write 1 Done Done Done false
162532024-02-15T03:04:45.484Z 7 Acked 1006 Read 6 Done Done Done false
162542024-02-15T03:04:45.484Z 8 Acked 1007 Read 4 Done Done Done false
162552024-02-15T03:04:45.484Z 9 Acked 1008 Write 4 Done Done Done false
162562024-02-15T03:04:45.484Z 10 Acked 1009 Write 7 Done Done Done false
162572024-02-15T03:04:45.484Z 11 Acked 1010 Read 3 Done Done Done false
162582024-02-15T03:04:45.484Z 12 Acked 1011 Write 1 Done Done Done false
162592024-02-15T03:04:45.484Z 13 Acked 1012 Write 6 Done Sent Done false
162602024-02-15T03:04:45.484Z 14 Acked 1013 Write 2 Done Sent Done false
162612024-02-15T03:04:45.484Z 15 Acked 1014 Write 4 Done Sent Done false
162622024-02-15T03:04:45.484Z 16 Acked 1015 Read 7 Done Sent Done false
162632024-02-15T03:04:45.484Z 17 Acked 1016 Write 4 Done Sent Done false
162642024-02-15T03:04:45.485Z 18 Acked 1017 Write 7 Done Sent Done false
162652024-02-15T03:04:45.485Z 19 Acked 1018 Read 6 Done Sent Done false
162662024-02-15T03:04:45.485Z 20 Acked 1019 Flush 0 Done Sent Done false
162672024-02-15T03:04:45.485Z 21 Acked 1020 Write 1 Done Sent Done false
162682024-02-15T03:04:45.485Z 22 Acked 1021 Flush 0 Done Sent Done false
162692024-02-15T03:04:45.485Z 23 Acked 1022 Write 8 Done Sent Done false
162702024-02-15T03:04:45.485Z 24 Acked 1023 Read 1 Done Sent Done false
162712024-02-15T03:04:45.485Z 25 Acked 1024 Write 9 Done Sent Done false
162722024-02-15T03:04:45.485Z 26 Acked 1025 Read 2 Done Sent Done false
162732024-02-15T03:04:45.485Z 27 Acked 1026 Read 3 Done Sent Done false
162742024-02-15T03:04:45.485Z 28 Acked 1027 Write 3 Done Sent Done false
162752024-02-15T03:04:45.485Z 29 Acked 1028 Read 10 Done Sent Done false
162762024-02-15T03:04:45.485Z 30 Acked 1029 Write 2 Sent Sent Sent false
162772024-02-15T03:04:45.485Z STATES DS:0 DS:1 DS:2 TOTAL
162782024-02-15T03:04:45.485Z New 0 0 0 0
162792024-02-15T03:04:45.485Z Sent 1 18 1 20
162802024-02-15T03:04:45.485Z Done 29 12 29 70
162812024-02-15T03:04:45.485Z Skipped 0 0 0 0
162822024-02-15T03:04:45.500Z Error 0 0 0 0
162832024-02-15T03:04:45.500ZLast Flush: 1021 0 1021
162842024-02-15T03:04:45.501ZDownstairs last five completed:
162852024-02-15T03:04:45.501ZUpstairs last five completed: 30 29 28 27 26
162862024-02-15T03:04:45.501Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:44.354792086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1725}
162872024-02-15T03:04:45.501ZWrote out file "/tmp/test_repair_verify.data"
162882024-02-15T03:04:45.501Z
162892024-02-15T03:04:45.501ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162902024-02-15T03:04:45.501ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162912024-02-15T03:04:45.501Z 0 000-019 138 138 138 227 227 227 T F T <---
162922024-02-15T03:04:45.501Z 1 020-039 146 139 146 238 230 238 F F F <---
162932024-02-15T03:04:45.501Z 2 040-059 144 144 144 236 236 236 F F F
162942024-02-15T03:04:45.502Z 3 060-079 142 142 142 235 235 235 F F F
162952024-02-15T03:04:45.502Z 4 080-099 146 145 146 239 237 239 F F F <---
162962024-02-15T03:04:45.502Z 5 100-119 129 129 129 214 214 214 T F T <---
162972024-02-15T03:04:45.502Z 6 120-139 145 145 145 237 237 237 F F F
162982024-02-15T03:04:45.502Z 7 140-159 145 145 145 237 237 237 F F F
162992024-02-15T03:04:45.502Z 8 160-179 146 145 146 238 237 238 F T F <---
163002024-02-15T03:04:45.502Z 9 180-199 145 145 145 237 237 237 F F F
163012024-02-15T03:04:45.502Z 10 200-219 142 142 142 235 235 235 F F F
163022024-02-15T03:04:45.502Z 11 220-239 145 145 145 237 237 237 F F F
163032024-02-15T03:04:45.502Z 12 240-259 145 145 145 237 237 237 F F F
163042024-02-15T03:04:45.502Z 13 260-279 142 142 142 235 235 235 F F F
163052024-02-15T03:04:45.502Z 14 280-299 144 144 144 236 236 236 T F T <---
163062024-02-15T03:04:45.502Z 15 300-319 136 136 136 225 225 225 F F F
163072024-02-15T03:04:45.502Z 16 320-339 146 136 146 238 225 238 F F F <---
163082024-02-15T03:04:45.502Z 17 340-359 146 145 146 238 237 238 F T F <---
163092024-02-15T03:04:45.503Z 18 360-379 144 144 144 236 236 236 F F F
163102024-02-15T03:04:45.503Z 19 380-399 144 144 144 236 236 236 F F F
163112024-02-15T03:04:45.503Z 20 400-419 146 142 146 238 235 238 F F F <---
163122024-02-15T03:04:45.503Z 21 420-439 139 139 139 230 230 230 F F F
163132024-02-15T03:04:45.503Z 22 440-459 138 138 138 229 229 229 F F F
163142024-02-15T03:04:45.503Z 23 460-479 146 142 146 238 235 238 T T T <---
163152024-02-15T03:04:45.503Z 24 480-499 145 145 145 237 237 237 F F F
163162024-02-15T03:04:45.503Z 25 500-519 146 141 146 238 234 238 F T F <---
163172024-02-15T03:04:45.503Z 26 520-539 146 136 146 238 225 238 F T F <---
163182024-02-15T03:04:45.503Z 27 540-559 142 142 142 235 235 235 F F F
163192024-02-15T03:04:45.503Z 28 560-579 142 142 142 235 235 235 F F F
163202024-02-15T03:04:45.503Z 29 580-599 136 136 136 225 225 225 F F F
163212024-02-15T03:04:45.503ZMax gen: 146, Max flush: 239
163222024-02-15T03:04:45.503ZError: Difference in extent metadata found!
163232024-02-15T03:04:45.503ZA Difference in extent metadata is expected here
163242024-02-15T03:04:45.503ZOn loop 73
163252024-02-15T03:04:45.503Z
163262024-02-15T03:04:45.503ZVerifying data now
163272024-02-15T03:04:51.125ZNo change after verify
163282024-02-15T03:04:51.125ZLoop: 73 Downstairs dump after verify (and repair):
163292024-02-15T03:04:51.132ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163302024-02-15T03:04:51.132Z 0 000-019 147 147 147 240 240 240 F F F
163312024-02-15T03:04:51.132Z 1 020-039 146 146 146 238 238 238 F F F
163322024-02-15T03:04:51.133Z 2 040-059 144 144 144 236 236 236 F F F
163332024-02-15T03:04:51.133Z 3 060-079 142 142 142 235 235 235 F F F
163342024-02-15T03:04:51.133Z 4 080-099 146 146 146 239 239 239 F F F
163352024-02-15T03:04:51.133Z 5 100-119 147 147 147 240 240 240 F F F
163362024-02-15T03:04:51.134Z 6 120-139 145 145 145 237 237 237 F F F
163372024-02-15T03:04:51.134Z 7 140-159 145 145 145 237 237 237 F F F
163382024-02-15T03:04:51.135Z 8 160-179 146 146 146 238 238 238 F F F
163392024-02-15T03:04:51.135Z 9 180-199 145 145 145 237 237 237 F F F
163402024-02-15T03:04:51.135Z 10 200-219 142 142 142 235 235 235 F F F
163412024-02-15T03:04:51.135Z 11 220-239 145 145 145 237 237 237 F F F
163422024-02-15T03:04:51.135Z 12 240-259 145 145 145 237 237 237 F F F
163432024-02-15T03:04:51.135Z 13 260-279 142 142 142 235 235 235 F F F
163442024-02-15T03:04:51.136Z 14 280-299 147 147 147 240 240 240 F F F
163452024-02-15T03:04:51.136Z 15 300-319 136 136 136 225 225 225 F F F
163462024-02-15T03:04:51.136Z 16 320-339 146 146 146 238 238 238 F F F
163472024-02-15T03:04:51.136Z 17 340-359 146 146 146 238 238 238 F F F
163482024-02-15T03:04:51.136Z 18 360-379 144 144 144 236 236 236 F F F
163492024-02-15T03:04:51.137Z 19 380-399 144 144 144 236 236 236 F F F
163502024-02-15T03:04:51.137Z 20 400-419 146 146 146 238 238 238 F F F
163512024-02-15T03:04:51.137Z 21 420-439 139 139 139 230 230 230 F F F
163522024-02-15T03:04:51.137Z 22 440-459 138 138 138 229 229 229 F F F
163532024-02-15T03:04:51.137Z 23 460-479 147 147 147 240 240 240 F F F
163542024-02-15T03:04:51.138Z 24 480-499 145 145 145 237 237 237 F F F
163552024-02-15T03:04:51.138Z 25 500-519 146 146 146 238 238 238 F F F
163562024-02-15T03:04:51.138Z 26 520-539 146 146 146 238 238 238 F F F
163572024-02-15T03:04:51.138Z 27 540-559 142 142 142 235 235 235 F F F
163582024-02-15T03:04:51.138Z 28 560-579 142 142 142 235 235 235 F F F
163592024-02-15T03:04:51.139Z 29 580-599 136 136 136 225 225 225 F F F
163602024-02-15T03:04:51.139ZMax gen: 147, Max flush: 240
163612024-02-15T03:04:51.139Z
163622024-02-15T03:04:51.139ZBegin loop 74
163632024-02-15T03:04:51.140ZDownstairs to restart: 0
163642024-02-15T03:04:51.165Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039250441Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733}
163652024-02-15T03:04:51.166Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.03949662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733}
163662024-02-15T03:04:51.166Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039534735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733}
163672024-02-15T03:04:51.166Z{"msg":"Crucible 63b0f1dc-52b2-4037-bd8b-3e355b7660c7 has session id: ae5642f6-7c5e-4cec-b948-12702561a46c","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039578969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733Crucible runtime is spawned
163682024-02-15T03:04:51.166Z,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163692024-02-15T03:04:51.166Z{"msg":"Upstairs opts: Upstairs UUID: 63b0f1dc-52b2-4037-bd8b-3e355b7660c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039634202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163702024-02-15T03:04:51.166Z{"msg":"Crucible stats registered with UUID: 63b0f1dc-52b2-4037-bd8b-3e355b7660c7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039671108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163712024-02-15T03:04:51.167Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039713752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733}
163722024-02-15T03:04:51.167Z{"msg":"63b0f1dc-52b2-4037-bd8b-3e355b7660c7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039748378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163732024-02-15T03:04:51.167Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039774645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163742024-02-15T03:04:51.167Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039801101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163752024-02-15T03:04:51.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039836827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163762024-02-15T03:04:51.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039870833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163772024-02-15T03:04:51.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039902139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163782024-02-15T03:04:51.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039925936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163792024-02-15T03:04:51.168Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.039959922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163802024-02-15T03:04:51.168Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040000187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163812024-02-15T03:04:51.168Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040033052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163822024-02-15T03:04:51.168Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:50.04013029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163832024-02-15T03:04:51.169Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:50.040295799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163842024-02-15T03:04:51.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040331645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163852024-02-15T03:04:51.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040361941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163862024-02-15T03:04:51.169Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:50.040409855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163872024-02-15T03:04:51.169Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040446751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163882024-02-15T03:04:51.169Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040469348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163892024-02-15T03:04:51.169Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040492845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163902024-02-15T03:04:51.169Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040895984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163912024-02-15T03:04:51.169Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040948098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163922024-02-15T03:04:52.402Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.040982693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163932024-02-15T03:04:52.403Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.041021459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163942024-02-15T03:04:52.403Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.04164804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163952024-02-15T03:04:52.403Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.041709322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163962024-02-15T03:04:52.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.042361051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163972024-02-15T03:04:52.403Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.042398906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163982024-02-15T03:04:52.403Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.042415304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
163992024-02-15T03:04:52.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.042436111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164002024-02-15T03:04:52.403Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.042472847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164012024-02-15T03:04:52.403Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:50.042487785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164022024-02-15T03:04:52.427Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.040890418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164032024-02-15T03:04:52.427Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.040984256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"io task","client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164042024-02-15T03:04:52.427Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.041783816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164052024-02-15T03:04:52.428Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.041819362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164062024-02-15T03:04:52.428Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.042327608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164072024-02-15T03:04:52.428Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.042923093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164082024-02-15T03:04:52.428Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.042978366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164092024-02-15T03:04:52.428Z{"msg":"[0]R flush_numbers[0..12]: [240, 238, 236, 235, 239, 240, 237, 237, 238, 237, 235, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.04302836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164102024-02-15T03:04:52.428Z{"msg":"[0]R generation[0..12]: [147, 146, 144, 142, 146, 147, 145, 145, 146, 145, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043050077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164112024-02-15T03:04:52.428Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043078674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164122024-02-15T03:04:52.428Z{"msg":"[1]R flush_numbers[0..12]: [240, 238, 236, 235, 239, 240, 237, 237, 238, 237, 235, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043098051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164132024-02-15T03:04:52.428Z{"msg":"Wait for a query_work_queue command to finish before sending IO
164142024-02-15T03:04:52.428Z[1]R generation[0..12]: [147, 146, 144, 142, 146, 147, 145, 145, 146, 145, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043126358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164152024-02-15T03:04:52.428Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043147145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164162024-02-15T03:04:52.428Z{"msg":"[2]R flush_numbers[0..12]: [240, 238, 236, 235, 239, 240, 237, 237, 238, 237, 235, 237]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043175132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164172024-02-15T03:04:52.429Z{"msg":"[2]R generation[0..12]: [147, 146, 144, 142, 146, 147, 145, 145, 146, 145, 142, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043194549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164182024-02-15T03:04:52.429Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043221576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164192024-02-15T03:04:52.429Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043248673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164202024-02-15T03:04:52.429Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.04326826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164212024-02-15T03:04:52.429Z{"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043293797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164222024-02-15T03:04:52.429Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043312564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"":"downstairs","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164232024-02-15T03:04:52.429Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043337421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164242024-02-15T03:04:52.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043353819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"0","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164252024-02-15T03:04:52.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043377996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"1","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164262024-02-15T03:04:52.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043391995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"client":"2","":"downstairs","session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164272024-02-15T03:04:52.429Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043411892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164282024-02-15T03:04:52.429Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043424761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164292024-02-15T03:04:52.429Z{"msg":"63b0f1dc-52b2-4037-bd8b-3e355b7660c7 is now active with session: ae5642f6-7c5e-4cec-b948-12702561a46c","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043445328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164302024-02-15T03:04:52.430Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043458246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733,"session_id":"ae5642f6-7c5e-4cec-b948-12702561a46c"}
164312024-02-15T03:04:52.430Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:51.043477864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733}
164322024-02-15T03:04:52.430ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
164332024-02-15T03:04:52.430ZLoading write count information from file "/tmp/test_repair_verify.data"
164342024-02-15T03:04:52.430ZRun Repair workload
164352024-02-15T03:04:52.430Z01/30 Write block 115 len 24576 data: 9 8 10 9 7 8
164362024-02-15T03:04:52.430Z02/30 Write block 29 len 28672 data: 8 10 11 11 14 14 16
164372024-02-15T03:04:52.430Z03/30 Write block 31 len 28672 data: 12 12 15 15 17 14 13
164382024-02-15T03:04:52.430Z04/30 Write block 561 len 8192 data: 12 12
164392024-02-15T03:04:52.430Z05/30 Read block 525 len 28672
164402024-02-15T03:04:52.430Z06/30 Read block 393 len 12288
164412024-02-15T03:04:52.430Z07/30 Write block 517 len 12288 data: 11 10 9
164422024-02-15T03:04:52.430Z08/30 Write block 452 len 16384 data: 6 5 6 8
164432024-02-15T03:04:52.431Z09/30 Read block 94 len 28672
164442024-02-15T03:04:52.431Z10/30 Write block 486 len 16384 data: 10 12 8 11
164452024-02-15T03:04:52.431Z11/30 Read block 479 len 12288
164462024-02-15T03:04:52.431Z12/30 Write block 515 len 40960 data: 11 13 12 11 10 7 5 4 5 4
164472024-02-15T03:04:52.431Z13/30 Read block 37 len 20480
164482024-02-15T03:04:52.431Z14/30 Write block 422 len 20480 data: 10 12 10 10 10
164492024-02-15T03:04:52.431Z15/30 Read block 522 len 8192
164502024-02-15T03:04:52.431Z16/30 Read block 402 len 24576
164512024-02-15T03:04:52.431Z17/30 Read block 309 len 20480
164522024-02-15T03:04:52.431Z18/30 Write block 42 len 40960 data: 12 12 14 16 12 14 12 11 9 7
164532024-02-15T03:04:52.431Z19/30 Write block 430 len 4096 data: 8
164542024-02-15T03:04:52.431Z20/30 Write block 353 len 4096 data: 15
164552024-02-15T03:04:52.432Z21/30 Read block 380 len 12288
164562024-02-15T03:04:52.432Z22/30 Write block 320 len 12288 data: 10 10 10
164572024-02-15T03:04:52.432Z23/30 Read block 216 len 4096
164582024-02-15T03:04:52.432Z24/30 Write block 221 len 20480 data: 13 12 10 10 10
164592024-02-15T03:04:52.432Z25/30 Write block 259 len 20480 data: 16 13 10 5 5
164602024-02-15T03:04:52.432Z26/30 Write block 559 len 36864 data: 17 17 13 13 12 12 14 10 10
164612024-02-15T03:04:52.432Z27/30 Read block 303 len 40960
164622024-02-15T03:04:52.432Z28/30 Read block 564 len 36864
164632024-02-15T03:04:52.432Z29/30 Write block 321 len 36864 data: 11 11 11 10 12 13 12 11 12
164642024-02-15T03:04:52.433Z30/30 Write block 573 len 16384 data: 14 15 15 17
164652024-02-15T03:04:52.433Z----------------------------------------------------------------
164662024-02-15T03:04:52.433Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
164672024-02-15T03:04:52.433ZStates: Active Active Active
164682024-02-15T03:04:52.433ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164692024-02-15T03:04:52.433Z 1 Acked 1000 Write 6 Done Done Done false
164702024-02-15T03:04:52.433Z 2 Acked 1001 Write 7 Done Done Done false
164712024-02-15T03:04:52.433Z 3 Acked 1002 Write 7 Done Done Done false
164722024-02-15T03:04:52.433Z 4 Acked 1003 Write 2 Done Done Done false
164732024-02-15T03:04:52.434Z 5 Acked 1004 Read 7 Sent Done Done false
164742024-02-15T03:04:52.434Z 6 Acked 1005 Read 3 Sent Done Done false
164752024-02-15T03:04:52.434Z 7 Acked 1006 Write 3 Sent Done Done false
164762024-02-15T03:04:52.434Z 8 Acked 1007 Write 4 Sent Done Done false
164772024-02-15T03:04:52.434Z 9 Acked 1008 Read 7 Sent Done Done false
164782024-02-15T03:04:52.434Z 10 Acked 1009 Write 4 Sent Done Done false
164792024-02-15T03:04:52.434Z 11 Acked 1010 Read 3 Sent Done Done false
164802024-02-15T03:04:52.434Z 12 Acked 1011 Write 10 Sent Done Done false
164812024-02-15T03:04:52.434Z 13 Acked 1012 Read 5 Sent Done Done false
164822024-02-15T03:04:52.434Z 14 Acked 1013 Write 5 Sent Done Done false
164832024-02-15T03:04:52.435Z 15 Acked 1014 Read 2 Sent Done Done false
164842024-02-15T03:04:52.435Z 16 Acked 1015 Read 6 Sent Done Done false
164852024-02-15T03:04:52.435Z 17 Acked 1016 Read 5 Sent Done Done false
164862024-02-15T03:04:52.435Z 18 Acked 1017 Write 10 Sent Done Done false
164872024-02-15T03:04:52.435Z 19 Acked 1018 Write 1 Sent Done Done false
164882024-02-15T03:04:52.435Z 20 Acked 1019 Write 1 Sent Done Done false
164892024-02-15T03:04:52.435Z 21 Acked 1020 Read 3 Sent Done Done false
164902024-02-15T03:04:52.435Z 22 Acked 1021 Write 3 Sent Done Done false
164912024-02-15T03:04:52.435Z 23 Acked 1022 Read 1 Sent Done Done false
164922024-02-15T03:04:52.436Z 24 Acked 1023 Write 5 Sent Done Done false
164932024-02-15T03:04:52.436Z 25 Acked 1024 Write 5 Sent Done Done false
164942024-02-15T03:04:52.436Z 26 Acked 1025 Write 9 Sent Done Done false
164952024-02-15T03:04:52.436Z 27 Acked 1026 Read 10 Sent Done Done false
164962024-02-15T03:04:52.436Z 28 Acked 1027 Read 9 Sent Done Done false
164972024-02-15T03:04:52.436Z 29 Acked 1028 Write 9 Sent Sent Sent false
164982024-02-15T03:04:52.436Z 30 Acked 1029 Write 4 Sent Sent Sent false
164992024-02-15T03:04:52.436Z STATES DS:0 DS:1 DS:2 TOTAL
165002024-02-15T03:04:52.437Z New 0 0 0 0
165012024-02-15T03:04:52.437Z Sent 26 2 2 30
165022024-02-15T03:04:52.464Z Done 4 28 28 60
165032024-02-15T03:04:52.465Z Skipped 0 0 0 0
165042024-02-15T03:04:52.465Z Error 0 0 0 0
165052024-02-15T03:04:52.465ZLast Flush: 0 0 0
165062024-02-15T03:04:52.466ZDownstairs last five completed:
165072024-02-15T03:04:52.466ZUpstairs last five completed: 30 29 28 27 26
165082024-02-15T03:04:52.466Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:51.070671527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1733}
165092024-02-15T03:04:52.466ZWrote out file "/tmp/test_repair_verify.data"
165102024-02-15T03:04:52.466Z
165112024-02-15T03:04:52.467ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
165122024-02-15T03:04:52.467ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165132024-02-15T03:04:52.467Z 0 000-019 147 147 147 240 240 240 F F F
165142024-02-15T03:04:52.467Z 1 020-039 146 146 146 238 238 238 T T T
165152024-02-15T03:04:52.467Z 2 040-059 144 144 144 236 236 236 F T T <---
165162024-02-15T03:04:52.467Z 3 060-079 142 142 142 235 235 235 F F F
165172024-02-15T03:04:52.467Z 4 080-099 146 146 146 239 239 239 F F F
165182024-02-15T03:04:52.468Z 5 100-119 147 147 147 240 240 240 T T T
165192024-02-15T03:04:52.468Z 6 120-139 145 145 145 237 237 237 T T T
165202024-02-15T03:04:52.468Z 7 140-159 145 145 145 237 237 237 F F F
165212024-02-15T03:04:52.468Z 8 160-179 146 146 146 238 238 238 F F F
165222024-02-15T03:04:52.468Z 9 180-199 145 145 145 237 237 237 F F F
165232024-02-15T03:04:52.468Z 10 200-219 142 142 142 235 235 235 F F F
165242024-02-15T03:04:52.468Z 11 220-239 145 145 145 237 237 237 F T T <---
165252024-02-15T03:04:52.468Z 12 240-259 145 145 145 237 237 237 F T T <---
165262024-02-15T03:04:52.468Z 13 260-279 142 142 142 235 235 235 F T T <---
165272024-02-15T03:04:52.468Z 14 280-299 147 147 147 240 240 240 F F F
165282024-02-15T03:04:52.468Z 15 300-319 136 136 136 225 225 225 F F F
165292024-02-15T03:04:52.468Z 16 320-339 146 146 146 238 238 238 F T T <---
165302024-02-15T03:04:52.468Z 17 340-359 146 146 146 238 238 238 F T T <---
165312024-02-15T03:04:52.468Z 18 360-379 144 144 144 236 236 236 F F F
165322024-02-15T03:04:52.469Z 19 380-399 144 144 144 236 236 236 F F F
165332024-02-15T03:04:52.469Z 20 400-419 146 146 146 238 238 238 F F F
165342024-02-15T03:04:52.469Z 21 420-439 139 139 139 230 230 230 F T T <---
165352024-02-15T03:04:52.469Z 22 440-459 138 138 138 229 229 229 F T T <---
165362024-02-15T03:04:52.469Z 23 460-479 147 147 147 240 240 240 F F F
165372024-02-15T03:04:52.469Z 24 480-499 145 145 145 237 237 237 F T T <---
165382024-02-15T03:04:52.469Z 25 500-519 146 146 146 238 238 238 F T T <---
165392024-02-15T03:04:52.469Z 26 520-539 146 146 146 238 238 238 F T T <---
165402024-02-15T03:04:52.469Z 27 540-559 142 142 142 235 235 235 F T T <---
165412024-02-15T03:04:52.469Z 28 560-579 142 142 142 235 235 235 T T T
165422024-02-15T03:04:52.469Z 29 580-599 136 136 136 225 225 225 F F F
165432024-02-15T03:04:52.469ZMax gen: 147, Max flush: 240
165442024-02-15T03:04:52.469ZError: Difference in extent metadata found!
165452024-02-15T03:04:52.469ZA Difference in extent metadata is expected here
165462024-02-15T03:04:52.470ZOn loop 74
165472024-02-15T03:04:52.470Z
165482024-02-15T03:04:52.470ZVerifying data now
165492024-02-15T03:04:58.118ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
165502024-02-15T03:04:58.121Z563c563,565
165512024-02-15T03:04:58.121Z< 16,
165522024-02-15T03:04:58.121Z---
165532024-02-15T03:04:58.121Z> 17,
165542024-02-15T03:04:58.121Z> 13,
165552024-02-15T03:04:58.121Z> 13,
165562024-02-15T03:04:58.121Z566,570c568,570
165572024-02-15T03:04:58.121Z< 11,
165582024-02-15T03:04:58.121Z< 11,
165592024-02-15T03:04:58.121Z< 13,
165602024-02-15T03:04:58.121Z< 9,
165612024-02-15T03:04:58.121Z< 9,
165622024-02-15T03:04:58.122Z---
165632024-02-15T03:04:58.122Z> 14,
165642024-02-15T03:04:58.122Z> 10,
165652024-02-15T03:04:58.122Z> 10,
165662024-02-15T03:04:58.122Z576d575
165672024-02-15T03:04:58.122Z< 13,
165682024-02-15T03:04:58.122Z578,579c577,578
165692024-02-15T03:04:58.122Z< 14,
165702024-02-15T03:04:58.122Z< 16,
165712024-02-15T03:04:58.122Z---
165722024-02-15T03:04:58.122Z> 15,
165732024-02-15T03:04:58.122Z> 15,
165742024-02-15T03:04:58.122Z580a580
165752024-02-15T03:04:58.122Z> 17,
165762024-02-15T03:04:58.122Zdiff found after verify
165772024-02-15T03:04:58.125ZLoop: 74 Downstairs dump after verify (and repair):
165782024-02-15T03:04:58.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165792024-02-15T03:04:58.138Z 0 000-019 147 147 147 240 240 240 F F F
165802024-02-15T03:04:58.138Z 1 020-039 148 148 148 241 241 241 F F F
165812024-02-15T03:04:58.138Z 2 040-059 148 148 148 241 241 241 F F F
165822024-02-15T03:04:58.139Z 3 060-079 142 142 142 235 235 235 F F F
165832024-02-15T03:04:58.139Z 4 080-099 146 146 146 239 239 239 F F F
165842024-02-15T03:04:58.145Z 5 100-119 148 148 148 241 241 241 F F F
165852024-02-15T03:04:58.145Z 6 120-139 148 148 148 241 241 241 F F F
165862024-02-15T03:04:58.145Z 7 140-159 145 145 145 237 237 237 F F F
165872024-02-15T03:04:58.145Z 8 160-179 146 146 146 238 238 238 F F F
165882024-02-15T03:04:58.145Z 9 180-199 145 145 145 237 237 237 F F F
165892024-02-15T03:04:58.145Z 10 200-219 142 142 142 235 235 235 F F F
165902024-02-15T03:04:58.145Z 11 220-239 148 148 148 241 241 241 F F F
165912024-02-15T03:04:58.145Z 12 240-259 148 148 148 241 241 241 F F F
165922024-02-15T03:04:58.145Z 13 260-279 148 148 148 241 241 241 F F F
165932024-02-15T03:04:58.145Z 14 280-299 147 147 147 240 240 240 F F F
165942024-02-15T03:04:58.145Z 15 300-319 136 136 136 225 225 225 F F F
165952024-02-15T03:04:58.145Z 16 320-339 148 148 148 241 241 241 F F F
165962024-02-15T03:04:58.145Z 17 340-359 148 148 148 241 241 241 F F F
165972024-02-15T03:04:58.145Z 18 360-379 144 144 144 236 236 236 F F F
165982024-02-15T03:04:58.145Z 19 380-399 144 144 144 236 236 236 F F F
165992024-02-15T03:04:58.145Z 20 400-419 146 146 146 238 238 238 F F F
166002024-02-15T03:04:58.145Z 21 420-439 148 148 148 241 241 241 F F F
166012024-02-15T03:04:58.145Z 22 440-459 148 148 148 241 241 241 F F F
166022024-02-15T03:04:58.146Z 23 460-479 147 147 147 240 240 240 F F F
166032024-02-15T03:04:58.146Z 24 480-499 148 148 148 241 241 241 F F F
166042024-02-15T03:04:58.146Z 25 500-519 148 148 148 241 241 241 F F F
166052024-02-15T03:04:58.146Z 26 520-539 148 148 148 241 241 241 F F F
166062024-02-15T03:04:58.146Z 27 540-559 148 148 148 241 241 241 F F F
166072024-02-15T03:04:58.146Z 28 560-579 148 148 148 241 241 241 F F F
166082024-02-15T03:04:58.146Z 29 580-599 136 136 136 225 225 225 F F F
166092024-02-15T03:04:58.146ZMax gen: 148, Max flush: 241
166102024-02-15T03:04:58.150Z
166112024-02-15T03:04:58.150ZBegin loop 75
166122024-02-15T03:04:58.150ZDownstairs to restart: 1
166132024-02-15T03:04:58.162Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.055632119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744}
166142024-02-15T03:04:58.168Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
166152024-02-15T03:04:58.168Z,"time":"2024-02-15T03:04:57.055984325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744}
166162024-02-15T03:04:58.168Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.05610168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744}
166172024-02-15T03:04:58.168Z{"msg":"Crucible 330c808d-10b8-4fa4-8024-ccb2f024b1cc has session id: d9e08f22-370a-45b1-8dae-016e484d25e3","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056175621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166182024-02-15T03:04:58.168Z{"msg":"Upstairs opts: Upstairs UUID: 330c808d-10b8-4fa4-8024-ccb2f024b1cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056234073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166192024-02-15T03:04:58.168Z{"msg":"Crucible stats registered with UUID: 330c808d-10b8-4fa4-8024-ccb2f024b1cc","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.05626553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166202024-02-15T03:04:58.168Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056290706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744}
166212024-02-15T03:04:58.168Z{"msg":"330c808d-10b8-4fa4-8024-ccb2f024b1cc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056312944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166222024-02-15T03:04:58.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056351839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166232024-02-15T03:04:58.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056379845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166242024-02-15T03:04:58.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056424339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166252024-02-15T03:04:58.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056464945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166262024-02-15T03:04:58.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056493001Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166272024-02-15T03:04:58.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056517948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166282024-02-15T03:04:58.168Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056556853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166292024-02-15T03:04:58.168Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056583739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166302024-02-15T03:04:58.168Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056620115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166312024-02-15T03:04:58.169Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:57.056650641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166322024-02-15T03:04:58.169Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:57.056702685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166332024-02-15T03:04:58.169Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:57.056774296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166342024-02-15T03:04:58.169Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056816521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166352024-02-15T03:04:58.169Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056854386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166362024-02-15T03:04:58.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056886632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166372024-02-15T03:04:58.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056939545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166382024-02-15T03:04:58.169Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.056968901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166392024-02-15T03:04:58.169Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.057449501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166402024-02-15T03:04:58.169Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.057496265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166412024-02-15T03:04:58.169Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.057525692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166422024-02-15T03:04:58.169Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.057554638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166432024-02-15T03:04:58.169Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.058065484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166442024-02-15T03:04:58.169Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.058126376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166452024-02-15T03:04:58.169Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.058589978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166462024-02-15T03:04:58.169Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.058635393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166472024-02-15T03:04:58.169Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.05865335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166482024-02-15T03:04:58.169Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.058667658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166492024-02-15T03:04:58.169Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.058682297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166502024-02-15T03:04:58.169Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:57.058704194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166512024-02-15T03:04:59.165Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.058356331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166522024-02-15T03:04:59.171Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.058470297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166532024-02-15T03:04:59.172Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.059714231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166542024-02-15T03:04:59.172Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.059752616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166552024-02-15T03:04:59.172Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.060287879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166562024-02-15T03:04:59.172Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.060869716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166572024-02-15T03:04:59.172Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.06091749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166582024-02-15T03:04:59.172Z{"msg":"[0]R flush_numbers[0..12]: [240, 241, 241, 235, 239, 241, 241, 237, 238, 237, 235, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.060948916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166592024-02-15T03:04:59.172Z{"msg":"[0]R generation[0..12]: [147, 148, 148, 142, 146, 148, 148, 145, 146, 145, 142, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.060978472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166602024-02-15T03:04:59.172Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061014368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166612024-02-15T03:04:59.172Z{"msg":"[1]R flush_numbers[0..12]: [240, 241, 241, 235, 239, 241, 241, 237, 238, 237, 235, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061042884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166622024-02-15T03:04:59.172Z{"msg":"[1]R generation[0..12]: [147, 148, 148, 142, 146, 148, 148, 145, 146, 145, 142, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061062102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166632024-02-15T03:04:59.172Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO
166642024-02-15T03:04:59.172Z,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061093178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166652024-02-15T03:04:59.172Z{"msg":"[2]R flush_numbers[0..12]: [240, 241, 241, 235, 239, 241, 241, 237, 238, 237, 235, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061111946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166662024-02-15T03:04:59.172Z{"msg":"[2]R generation[0..12]: [147, 148, 148, 142, 146, 148, 148, 145, 146, 145, 142, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061130223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166672024-02-15T03:04:59.172Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061148281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166682024-02-15T03:04:59.172Z{"msg":"Max found gen is 149","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061181737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166692024-02-15T03:04:59.172Z{"msg":"Generation requested: 150 >= found:149","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061199295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166702024-02-15T03:04:59.172Z{"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061231451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166712024-02-15T03:04:59.172Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061248438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"":"downstairs","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166722024-02-15T03:04:59.172Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061266056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166732024-02-15T03:04:59.172Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061296883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"0","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166742024-02-15T03:04:59.172Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.06131413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"1","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166752024-02-15T03:04:59.172Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061338867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"client":"2","":"downstairs","session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166762024-02-15T03:04:59.173Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061355785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166772024-02-15T03:04:59.173Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061380162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166782024-02-15T03:04:59.173Z{"msg":"330c808d-10b8-4fa4-8024-ccb2f024b1cc is now active with session: d9e08f22-370a-45b1-8dae-016e484d25e3","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.06139708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166792024-02-15T03:04:59.173Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061421357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744,"session_id":"d9e08f22-370a-45b1-8dae-016e484d25e3"}
166802024-02-15T03:04:59.173Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:04:58.061437615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744}
166812024-02-15T03:04:59.173ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
166822024-02-15T03:04:59.173ZLoading write count information from file "/tmp/test_repair_verify.data"
166832024-02-15T03:04:59.173ZRun Repair workload
166842024-02-15T03:04:59.173Z01/30 Flush
166852024-02-15T03:04:59.177Z02/30 Write block 195 len 36864 data: 18 14 12 9 11 12 10 7 6
166862024-02-15T03:04:59.177Z03/30 Read block 484 len 40960
166872024-02-15T03:04:59.177Z04/30 Write block 451 len 4096 data: 7
166882024-02-15T03:04:59.177Z05/30 Read block 232 len 8192
166892024-02-15T03:04:59.177Z06/30 Read block 480 len 40960
166902024-02-15T03:04:59.177Z07/30 Write block 462 len 36864 data: 10 7 7 9 12 11 11 14 16
166912024-02-15T03:04:59.177Z08/30 Read block 109 len 4096
166922024-02-15T03:04:59.180Z09/30 Write block 205 len 12288 data: 8 8 9
166932024-02-15T03:04:59.180Z10/30 Read block 301 len 20480
166942024-02-15T03:04:59.180Z11/30 Write block 84 len 16384 data: 9 8 11 11
166952024-02-15T03:04:59.181Z12/30 Read block 432 len 8192
166962024-02-15T03:04:59.183Z13/30 Write block 151 len 28672 data: 10 9 11 11 11 16 16
166972024-02-15T03:04:59.183Z14/30 Write block 552 len 12288 data: 6 9 11
166982024-02-15T03:04:59.184Z15/30 Write block 453 len 32768 data: 6 7 9 8 7 8 8 11
166992024-02-15T03:04:59.184Z16/30 Write block 458 len 40960 data: 9 9 12 11 11 8 8 10 13 12
167002024-02-15T03:04:59.184Z17/30 Flush
167012024-02-15T03:04:59.187Z18/30 Write block 63 len 28672 data: 13 14 10 10 11 11 11
167022024-02-15T03:04:59.190Z19/30 Write block 67 len 4096 data: 12
167032024-02-15T03:04:59.190Z20/30 Write block 28 len 16384 data: 10 9 11 13
167042024-02-15T03:04:59.190Z21/30 Read block 545 len 36864
167052024-02-15T03:04:59.190Z22/30 Write block 565 len 4096 data: 14
167062024-02-15T03:04:59.193Z23/30 Read block 198 len 16384
167072024-02-15T03:04:59.193Z24/30 Read block 231 len 8192
167082024-02-15T03:04:59.193Z25/30 Read block 278 len 4096
167092024-02-15T03:04:59.193Z26/30 Read block 565 len 8192
167102024-02-15T03:04:59.193Z27/30 Write block 41 len 4096 data: 11
167112024-02-15T03:04:59.206Z28/30 Write block 531 len 32768 data: 9 12 10 10 12 9 12 10
167122024-02-15T03:04:59.206Z29/30 Write block 55 len 24576 data: 12 11 14 14 11 12
167132024-02-15T03:04:59.206Z30/30 Write block 500 len 28672 data: 13 13 14 15 15 13 12
167142024-02-15T03:04:59.207Z----------------------------------------------------------------
167152024-02-15T03:04:59.207Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:29
167162024-02-15T03:04:59.207ZStates: Active Active Active
167172024-02-15T03:04:59.207ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167182024-02-15T03:04:59.207Z 2 Acked 1001 Write 9 Done Done Done false
167192024-02-15T03:04:59.207Z 3 Acked 1002 Read 10 Done Done Done false
167202024-02-15T03:04:59.207Z 4 Acked 1003 Write 1 Done Done Done false
167212024-02-15T03:04:59.207Z 5 Acked 1004 Read 2 Done Sent Done false
167222024-02-15T03:04:59.207Z 6 Acked 1005 Read 10 Done Sent Done false
167232024-02-15T03:04:59.207Z 7 Acked 1006 Write 9 Done Sent Done false
167242024-02-15T03:04:59.207Z 8 Acked 1007 Read 1 Done Sent Done false
167252024-02-15T03:04:59.207Z 9 Acked 1008 Write 3 Done Sent Done false
167262024-02-15T03:04:59.207Z 10 Acked 1009 Read 5 Done Sent Done false
167272024-02-15T03:04:59.207Z 11 Acked 1010 Write 4 Done Sent Done false
167282024-02-15T03:04:59.207Z 12 Acked 1011 Read 2 Done Sent Done false
167292024-02-15T03:04:59.207Z 13 Acked 1012 Write 7 Done Sent Done false
167302024-02-15T03:04:59.207Z 14 Acked 1013 Write 3 Done Sent Done false
167312024-02-15T03:04:59.207Z 15 Acked 1014 Write 8 Done Sent Done false
167322024-02-15T03:04:59.207Z 16 Acked 1015 Write 10 Done Sent Done false
167332024-02-15T03:04:59.208Z 17 Acked 1016 Flush 0 Done Sent Done false
167342024-02-15T03:04:59.208Z 18 Acked 1017 Write 7 Done Sent Done false
167352024-02-15T03:04:59.208Z 19 Acked 1018 Write 1 Done Sent Done false
167362024-02-15T03:04:59.208Z 20 Acked 1019 Write 4 Done Sent Done false
167372024-02-15T03:04:59.208Z 21 Acked 1020 Read 9 Done Sent Done false
167382024-02-15T03:04:59.208Z 22 Acked 1021 Write 1 Done Sent Done false
167392024-02-15T03:04:59.208Z 23 Acked 1022 Read 4 Done Sent Done false
167402024-02-15T03:04:59.208Z 24 Acked 1023 Read 2 Done Sent Done false
167412024-02-15T03:04:59.208Z 25 Acked 1024 Read 1 Done Sent Done false
167422024-02-15T03:04:59.208Z 26 Acked 1025 Read 2 Done Sent Done false
167432024-02-15T03:04:59.208Z 27 Acked 1026 Write 1 Done Sent Done false
167442024-02-15T03:04:59.208Z 28 Acked 1027 Write 8 Done Sent Done false
167452024-02-15T03:04:59.208Z 29 Acked 1028 Write 6 Done Sent Done false
167462024-02-15T03:04:59.208Z 30 Acked 1029 Write 7 Sent Sent Sent false
167472024-02-15T03:04:59.208Z STATES DS:0 DS:1 DS:2 TOTAL
167482024-02-15T03:04:59.208Z New 0 0 0 0
167492024-02-15T03:04:59.208Z Sent 1 26 1 28
167502024-02-15T03:04:59.208Z Done 28 3 28 59
167512024-02-15T03:04:59.208Z Skipped 0 0 0 0
167522024-02-15T03:04:59.208Z Error 0 0 0 0
167532024-02-15T03:04:59.209ZLast Flush: 1016 1000 1016
167542024-02-15T03:04:59.209ZDownstairs last five completed: 1000
167552024-02-15T03:04:59.209ZUpstairs last five completed: 30 29 28 27 26
167562024-02-15T03:04:59.209Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:04:58.0894099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1744}
167572024-02-15T03:04:59.209ZWrote out file "/tmp/test_repair_verify.data"
167582024-02-15T03:04:59.221Z
167592024-02-15T03:04:59.221ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
167602024-02-15T03:04:59.643ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167612024-02-15T03:04:59.643Z 0 000-019 147 147 147 240 240 240 F F F
167622024-02-15T03:04:59.643Z 1 020-039 148 148 148 241 241 241 T F T <---
167632024-02-15T03:04:59.643Z 2 040-059 148 148 148 241 241 241 T F T <---
167642024-02-15T03:04:59.643Z 3 060-079 142 142 142 235 235 235 T F T <---
167652024-02-15T03:04:59.644Z 4 080-099 150 146 150 243 239 243 F F F <---
167662024-02-15T03:04:59.644Z 5 100-119 148 148 148 241 241 241 F F F
167672024-02-15T03:04:59.644Z 6 120-139 148 148 148 241 241 241 F F F
167682024-02-15T03:04:59.644Z 7 140-159 150 145 150 243 237 243 F F F <---
167692024-02-15T03:04:59.644Z 8 160-179 146 146 146 238 238 238 F F F
167702024-02-15T03:04:59.644Z 9 180-199 150 145 150 243 237 243 F T F <---
167712024-02-15T03:04:59.644Z 10 200-219 150 142 150 243 235 243 F T F <---
167722024-02-15T03:04:59.645Z 11 220-239 148 148 148 241 241 241 F F F
167732024-02-15T03:04:59.645Z 12 240-259 148 148 148 241 241 241 F F F
167742024-02-15T03:04:59.645Z 13 260-279 148 148 148 241 241 241 F F F
167752024-02-15T03:04:59.645Z 14 280-299 147 147 147 240 240 240 F F F
167762024-02-15T03:04:59.645Z 15 300-319 136 136 136 225 225 225 F F F
167772024-02-15T03:04:59.645Z 16 320-339 148 148 148 241 241 241 F F F
167782024-02-15T03:04:59.645Z 17 340-359 148 148 148 241 241 241 F F F
167792024-02-15T03:04:59.645Z 18 360-379 144 144 144 236 236 236 F F F
167802024-02-15T03:04:59.645Z 19 380-399 144 144 144 236 236 236 F F F
167812024-02-15T03:04:59.645Z 20 400-419 146 146 146 238 238 238 F F F
167822024-02-15T03:04:59.646Z 21 420-439 148 148 148 241 241 241 F F F
167832024-02-15T03:04:59.646Z 22 440-459 150 148 150 243 241 243 F T F <---
167842024-02-15T03:04:59.646Z 23 460-479 150 147 150 243 240 243 F F F <---
167852024-02-15T03:04:59.646Z 24 480-499 148 148 148 241 241 241 F F F
167862024-02-15T03:04:59.646Z 25 500-519 148 148 148 241 241 241 T F T <---
167872024-02-15T03:04:59.646Z 26 520-539 148 148 148 241 241 241 T F T <---
167882024-02-15T03:04:59.646Z 27 540-559 150 148 150 243 241 243 F F F <---
167892024-02-15T03:04:59.646Z 28 560-579 148 148 148 241 241 241 T F T <---
167902024-02-15T03:04:59.646Z 29 580-599 136 136 136 225 225 225 F F F
167912024-02-15T03:04:59.647ZMax gen: 150, Max flush: 243
167922024-02-15T03:04:59.647ZError: Difference in extent metadata found!
167932024-02-15T03:04:59.668ZA Difference in extent metadata is expected here
167942024-02-15T03:04:59.668ZOn loop 75
167952024-02-15T03:04:59.668Z
167962024-02-15T03:04:59.668ZVerifying data now
167972024-02-15T03:05:04.899ZNo change after verify
167982024-02-15T03:05:04.900ZLoop: 75 Downstairs dump after verify (and repair):
167992024-02-15T03:05:05.327ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168002024-02-15T03:05:05.327Z 0 000-019 147 147 147 240 240 240 F F F
168012024-02-15T03:05:05.328Z 1 020-039 151 151 151 244 244 244 F F F
168022024-02-15T03:05:05.328Z 2 040-059 151 151 151 244 244 244 F F F
168032024-02-15T03:05:05.328Z 3 060-079 151 151 151 244 244 244 F F F
168042024-02-15T03:05:05.328Z 4 080-099 150 150 150 243 243 243 F F F
168052024-02-15T03:05:05.328Z 5 100-119 148 148 148 241 241 241 F F F
168062024-02-15T03:05:05.328Z 6 120-139 148 148 148 241 241 241 F F F
168072024-02-15T03:05:05.328Z 7 140-159 150 150 150 243 243 243 F F F
168082024-02-15T03:05:05.329Z 8 160-179 146 146 146 238 238 238 F F F
168092024-02-15T03:05:05.329Z 9 180-199 150 150 150 243 243 243 F F F
168102024-02-15T03:05:05.329Z 10 200-219 150 150 150 243 243 243 F F F
168112024-02-15T03:05:05.329Z 11 220-239 148 148 148 241 241 241 F F F
168122024-02-15T03:05:05.329Z 12 240-259 148 148 148 241 241 241 F F F
168132024-02-15T03:05:05.329Z 13 260-279 148 148 148 241 241 241 F F F
168142024-02-15T03:05:05.329Z 14 280-299 147 147 147 240 240 240 F F F
168152024-02-15T03:05:05.337Z 15 300-319 136 136 136 225 225 225 F F F
168162024-02-15T03:05:05.337Z 16 320-339 148 148 148 241 241 241 F F F
168172024-02-15T03:05:05.338Z 17 340-359 148 148 148 241 241 241 F F F
168182024-02-15T03:05:05.338Z 18 360-379 144 144 144 236 236 236 F F F
168192024-02-15T03:05:05.338Z 19 380-399 144 144 144 236 236 236 F F F
168202024-02-15T03:05:05.338Z 20 400-419 146 146 146 238 238 238 F F F
168212024-02-15T03:05:05.338Z 21 420-439 148 148 148 241 241 241 F F F
168222024-02-15T03:05:05.339Z 22 440-459 150 150 150 243 243 243 F F F
168232024-02-15T03:05:05.339Z 23 460-479 150 150 150 243 243 243 F F F
168242024-02-15T03:05:05.339Z 24 480-499 148 148 148 241 241 241 F F F
168252024-02-15T03:05:05.339Z 25 500-519 151 151 151 244 244 244 F F F
168262024-02-15T03:05:05.339Z 26 520-539 151 151 151 244 244 244 F F F
168272024-02-15T03:05:05.339Z 27 540-559 150 150 150 243 243 243 F F F
168282024-02-15T03:05:05.339Z 28 560-579 151 151 151 244 244 244 F F F
168292024-02-15T03:05:05.339Z 29 580-599 136 136 136 225 225 225 F F F
168302024-02-15T03:05:05.339ZMax gen: 151, Max flush: 244
168312024-02-15T03:05:05.339Z
168322024-02-15T03:05:05.339ZBegin loop 76
168332024-02-15T03:05:05.340ZDownstairs to restart: 2
168342024-02-15T03:05:05.340Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833163455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752}
168352024-02-15T03:05:05.340Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
168362024-02-15T03:05:05.340Z,"time":"2024-02-15T03:05:03.833481406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752}
168372024-02-15T03:05:05.340Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833574354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752}
168382024-02-15T03:05:05.340Z{"msg":"Crucible f039b390-c535-44e7-9e4c-0db624176087 has session id: e800998b-c008-4290-9c2e-65acc8a78b7d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833627897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168392024-02-15T03:05:05.340Z{"msg":"Upstairs opts: Upstairs UUID: f039b390-c535-44e7-9e4c-0db624176087, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833690639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168402024-02-15T03:05:05.340Z{"msg":"Crucible stats registered with UUID: f039b390-c535-44e7-9e4c-0db624176087","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833739113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168412024-02-15T03:05:05.340Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833810854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752}
168422024-02-15T03:05:05.340Z{"msg":"f039b390-c535-44e7-9e4c-0db624176087 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833869647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168432024-02-15T03:05:05.340Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833952487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168442024-02-15T03:05:05.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.833984752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168452024-02-15T03:05:05.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834022508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168462024-02-15T03:05:05.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834067222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168472024-02-15T03:05:05.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834093039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168482024-02-15T03:05:05.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834120755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168492024-02-15T03:05:05.341Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834137213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168502024-02-15T03:05:05.341Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.83416369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168512024-02-15T03:05:05.341Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834217313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168522024-02-15T03:05:05.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.83424506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168532024-02-15T03:05:05.341Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:03.834273726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168542024-02-15T03:05:05.342Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:03.834316231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168552024-02-15T03:05:05.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834386452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168562024-02-15T03:05:05.342Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:03.834419858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168572024-02-15T03:05:05.342Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834434146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168582024-02-15T03:05:05.342Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834447725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168592024-02-15T03:05:05.342Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.834594876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168602024-02-15T03:05:05.342Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.835048889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168612024-02-15T03:05:05.342Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.835080085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168622024-02-15T03:05:05.342Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.835137888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168632024-02-15T03:05:05.342Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.835186372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168642024-02-15T03:05:05.342Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.835626037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168652024-02-15T03:05:05.342Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.835747162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168662024-02-15T03:05:05.343Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.836180877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168672024-02-15T03:05:05.343Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.836227422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168682024-02-15T03:05:05.343Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.836246939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168692024-02-15T03:05:05.343Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.836261967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168702024-02-15T03:05:05.343Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.836276865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168712024-02-15T03:05:05.343Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:03.836291094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168722024-02-15T03:05:05.945Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.83514608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168732024-02-15T03:05:05.945Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.835222091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168742024-02-15T03:05:05.953Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.836169012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168752024-02-15T03:05:05.953Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.836208377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168762024-02-15T03:05:05.953Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.836756439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168772024-02-15T03:05:05.953Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837388719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168782024-02-15T03:05:05.954Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837419595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168792024-02-15T03:05:05.954Z{"msg":"[0]R flush_numbers[0..12]: [240, 244, 244, 244, 243, 241, 241, 243, 238, 243, 243, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837435573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168802024-02-15T03:05:05.954Z{"msg":"[0]R generation[0..12]: [147, 151, 151, 151, 150, 148, 148, 150, 146, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837450152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168812024-02-15T03:05:05.954Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837473969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168822024-02-15T03:05:05.954Z{"msg":"[1]R flush_numbers[0..12]: [240, 244, 244, 244, 243, 241, 241, 243, 238, 243, 243, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837499805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168832024-02-15T03:05:05.954Z{"msg":"[1]R generation[0..12]: [147, 151, 151, 151, 150, 148, 148, 150, 146, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837531101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168842024-02-15T03:05:05.954Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837547509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168852024-02-15T03:05:05.954Z{"msg":"[2]R flush_numbers[0..12]: [240, 244, 244, 244, 243, 241, 241, 243, 238, 243, 243, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837580675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168862024-02-15T03:05:05.954ZWait for a query_work_queue command to finish before sending IO
168872024-02-15T03:05:05.954Z{"msg":"[2]R generation[0..12]: [147, 151, 151, 151, 150, 148, 148, 150, 146, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.83762086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168882024-02-15T03:05:05.954Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837641308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168892024-02-15T03:05:05.955Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837671074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168902024-02-15T03:05:05.955Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837686452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168912024-02-15T03:05:05.955Z{"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.83770594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168922024-02-15T03:05:05.955Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837733986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"":"downstairs","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168932024-02-15T03:05:05.955Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837750104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168942024-02-15T03:05:05.955Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.83777915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"0","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168952024-02-15T03:05:05.955Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837796818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"1","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168962024-02-15T03:05:05.955Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837826984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"client":"2","":"downstairs","session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168972024-02-15T03:05:05.955Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837845132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168982024-02-15T03:05:05.955Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837873199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
168992024-02-15T03:05:05.955Z{"msg":"f039b390-c535-44e7-9e4c-0db624176087 is now active with session: e800998b-c008-4290-9c2e-65acc8a78b7d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837893036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
169002024-02-15T03:05:05.956Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837919223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752,"session_id":"e800998b-c008-4290-9c2e-65acc8a78b7d"}
169012024-02-15T03:05:05.956Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:04.837935691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752}
169022024-02-15T03:05:05.956ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169032024-02-15T03:05:05.956ZLoading write count information from file "/tmp/test_repair_verify.data"
169042024-02-15T03:05:05.956ZRun Repair workload
169052024-02-15T03:05:05.956Z01/30 Write block 482 len 20480 data: 12 8 10 8 11
169062024-02-15T03:05:05.956Z02/30 Read block 396 len 40960
169072024-02-15T03:05:05.956Z03/30 Write block 401 len 20480 data: 10 9 8 8 13
169082024-02-15T03:05:05.956Z04/30 Read block 586 len 32768
169092024-02-15T03:05:05.967Z05/30 Flush
169102024-02-15T03:05:05.967Z06/30 Flush
169112024-02-15T03:05:05.967Z07/30 Write block 315 len 20480 data: 10 11 12 11 10
169122024-02-15T03:05:05.967Z08/30 Write block 70 len 12288 data: 11 12 11
169132024-02-15T03:05:05.967Z09/30 Flush
169142024-02-15T03:05:05.967Z10/30 Write block 111 len 28672 data: 9 7 6 7 10 9 11
169152024-02-15T03:05:05.968Z11/30 Write block 467 len 20480 data: 13 12 15 17 13
169162024-02-15T03:05:05.968Z12/30 Write block 23 len 12288 data: 4 6 6
169172024-02-15T03:05:05.968Z13/30 Read block 584 len 40960
169182024-02-15T03:05:05.968Z14/30 Write block 294 len 36864 data: 14 12 11 11 11 11 10 12 12
169192024-02-15T03:05:05.968Z15/30 Write block 125 len 28672 data: 7 9 9 9 9 9 10
169202024-02-15T03:05:05.975Z16/30 Write block 379 len 4096 data: 11
169212024-02-15T03:05:05.975Z17/30 Write block 38 len 12288 data: 14 12 10
169222024-02-15T03:05:05.976Z18/30 Read block 461 len 32768
169232024-02-15T03:05:05.976Z19/30 Write block 275 len 12288 data: 9 10 12
169242024-02-15T03:05:05.976Z20/30 Write block 237 len 8192 data: 14 13
169252024-02-15T03:05:05.976Z21/30 Read block 10 len 4096
169262024-02-15T03:05:05.976Z22/30 Write block 247 len 32768 data: 11 9 10 11 11 13 12 12
169272024-02-15T03:05:05.976Z23/30 Read block 497 len 28672
169282024-02-15T03:05:05.977Z24/30 Read block 215 len 24576
169292024-02-15T03:05:05.977Z25/30 Read block 452 len 16384
169302024-02-15T03:05:05.977Z26/30 Write block 205 len 32768 data: 9 9 10 11 10 8 9 11
169312024-02-15T03:05:05.977Z27/30 Write block 259 len 32768 data: 17 14 11 6 6 6 9 9
169322024-02-15T03:05:05.977Z28/30 Read block 3 len 4096
169332024-02-15T03:05:05.986Z29/30 Write block 24 len 28672 data: 7 7 8 9 11 10 12
169342024-02-15T03:05:05.987Z30/30 Read block 139 len 12288
169352024-02-15T03:05:05.987Z----------------------------------------------------------------
169362024-02-15T03:05:05.987Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
169372024-02-15T03:05:05.987ZStates: Active Active Active
169382024-02-15T03:05:05.987ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169392024-02-15T03:05:05.987Z 1 Acked 1000 Write 5 Done Done Done false
169402024-02-15T03:05:05.987Z 2 Acked 1001 Read 10 Done Done Done false
169412024-02-15T03:05:05.988Z 3 Acked 1002 Write 5 Done Done Sent false
169422024-02-15T03:05:05.988Z 4 Acked 1003 Read 8 Done Done Sent false
169432024-02-15T03:05:05.988Z 5 Acked 1004 Flush 0 Done Done Sent false
169442024-02-15T03:05:05.988Z 6 Acked 1005 Flush 0 Done Done Sent false
169452024-02-15T03:05:05.988Z 7 Acked 1006 Write 5 Done Done Sent false
169462024-02-15T03:05:05.988Z 8 Acked 1007 Write 3 Done Done Sent false
169472024-02-15T03:05:05.988Z 9 Acked 1008 Flush 0 Done Done Sent false
169482024-02-15T03:05:05.988Z 10 Acked 1009 Write 7 Done Done Sent false
169492024-02-15T03:05:05.989Z 11 Acked 1010 Write 5 Done Done Sent false
169502024-02-15T03:05:05.989Z 12 Acked 1011 Write 3 Done Done Sent false
169512024-02-15T03:05:05.989Z 13 Acked 1012 Read 10 Done Done Sent false
169522024-02-15T03:05:05.989Z 14 Acked 1013 Write 9 Done Done Sent false
169532024-02-15T03:05:05.989Z 15 Acked 1014 Write 7 Done Done Sent false
169542024-02-15T03:05:05.990Z 16 Acked 1015 Write 1 Done Done Sent false
169552024-02-15T03:05:05.990Z 17 Acked 1016 Write 3 Done Done Sent false
169562024-02-15T03:05:05.990Z 18 Acked 1017 Read 8 Done Done Sent false
169572024-02-15T03:05:05.990Z 19 Acked 1018 Write 3 Done Done Sent false
169582024-02-15T03:05:05.990Z 20 Acked 1019 Write 2 Done Done Sent false
169592024-02-15T03:05:05.991Z 21 Acked 1020 Read 1 Done Done Sent false
169602024-02-15T03:05:05.991Z 22 Acked 1021 Write 8 Done Done Sent false
169612024-02-15T03:05:05.991Z 23 Acked 1022 Read 7 Done Done Sent false
169622024-02-15T03:05:05.991Z 24 Acked 1023 Read 6 Done Done Sent false
169632024-02-15T03:05:05.992Z 25 Acked 1024 Read 4 Done Done Sent false
169642024-02-15T03:05:05.992Z 26 Acked 1025 Write 8 Done Done Sent false
169652024-02-15T03:05:05.995Z 27 Acked 1026 Write 8 Done Done Sent false
169662024-02-15T03:05:05.995Z 28 Acked 1027 Read 1 Done Done Sent false
169672024-02-15T03:05:05.995Z 29 Acked 1028 Write 7 Done Done Sent false
169682024-02-15T03:05:05.995Z 30 Acked 1029 Read 3 Sent Done Sent false
169692024-02-15T03:05:05.995Z STATES DS:0 DS:1 DS:2 TOTAL
169702024-02-15T03:05:05.995Z New 0 0 0 0
169712024-02-15T03:05:05.995Z Sent 1 0 28 29
169722024-02-15T03:05:05.996Z Done 29 30 2 61
169732024-02-15T03:05:05.996Z Skipped 0 0 0 0
169742024-02-15T03:05:05.996Z Error 0 0 0 0
169752024-02-15T03:05:05.996ZLast Flush: 1008 1008 0
169762024-02-15T03:05:05.996ZDownstairs last five completed:
169772024-02-15T03:05:05.996ZUpstairs last five completed: 30 29 28 27 26
169782024-02-15T03:05:05.997Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:04.868071095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1752}
169792024-02-15T03:05:05.997ZWrote out file "/tmp/test_repair_verify.data"
169802024-02-15T03:05:05.997Z
169812024-02-15T03:05:06.015ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
169822024-02-15T03:05:06.015ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169832024-02-15T03:05:06.015Z 0 000-019 147 147 147 240 240 240 F F F
169842024-02-15T03:05:06.015Z 1 020-039 151 151 151 244 244 244 T T F <---
169852024-02-15T03:05:06.015Z 2 040-059 151 151 151 244 244 244 T T F <---
169862024-02-15T03:05:06.015Z 3 060-079 152 152 151 247 247 244 F F F <---
169872024-02-15T03:05:06.016Z 4 080-099 150 150 150 243 243 243 F F F
169882024-02-15T03:05:06.016Z 5 100-119 148 148 148 241 241 241 T T F <---
169892024-02-15T03:05:06.016Z 6 120-139 148 148 148 241 241 241 T T F <---
169902024-02-15T03:05:06.016Z 7 140-159 150 150 150 243 243 243 F F F
169912024-02-15T03:05:06.016Z 8 160-179 146 146 146 238 238 238 F F F
169922024-02-15T03:05:06.016Z 9 180-199 150 150 150 243 243 243 F F F
169932024-02-15T03:05:06.016Z 10 200-219 150 150 150 243 243 243 T T F <---
169942024-02-15T03:05:06.017Z 11 220-239 148 148 148 241 241 241 T T F <---
169952024-02-15T03:05:06.017Z 12 240-259 148 148 148 241 241 241 T T F <---
169962024-02-15T03:05:06.017Z 13 260-279 148 148 148 241 241 241 T T F <---
169972024-02-15T03:05:06.017Z 14 280-299 147 147 147 240 240 240 T T F <---
169982024-02-15T03:05:06.017Z 15 300-319 152 152 136 247 247 225 T T F <---
169992024-02-15T03:05:06.017Z 16 320-339 148 148 148 241 241 241 F F F
170002024-02-15T03:05:06.017Z 17 340-359 148 148 148 241 241 241 F F F
170012024-02-15T03:05:06.018Z 18 360-379 144 144 144 236 236 236 T T F <---
170022024-02-15T03:05:06.018Z 19 380-399 144 144 144 236 236 236 F F F
170032024-02-15T03:05:06.018Z 20 400-419 152 152 146 245 245 238 F F F <---
170042024-02-15T03:05:06.018Z 21 420-439 148 148 148 241 241 241 F F F
170052024-02-15T03:05:06.018Z 22 440-459 150 150 150 243 243 243 F F F
170062024-02-15T03:05:06.018Z 23 460-479 150 150 150 243 243 243 T T F <---
170072024-02-15T03:05:06.018Z 24 480-499 152 152 148 245 245 241 F F T <---
170082024-02-15T03:05:06.018Z 25 500-519 151 151 151 244 244 244 F F F
170092024-02-15T03:05:06.019Z 26 520-539 151 151 151 244 244 244 F F F
170102024-02-15T03:05:06.019Z 27 540-559 150 150 150 243 243 243 F F F
170112024-02-15T03:05:06.019Z 28 560-579 151 151 151 244 244 244 F F F
170122024-02-15T03:05:06.019Z 29 580-599 136 136 136 225 225 225 F F F
170132024-02-15T03:05:06.019ZMax gen: 152, Max flush: 247
170142024-02-15T03:05:06.019ZError: Difference in extent metadata found!
170152024-02-15T03:05:06.025ZA Difference in extent metadata is expected here
170162024-02-15T03:05:06.025ZOn loop 76
170172024-02-15T03:05:06.025Z
170182024-02-15T03:05:06.025ZVerifying data now
170192024-02-15T03:05:11.785ZNo change after verify
170202024-02-15T03:05:11.785ZLoop: 76 Downstairs dump after verify (and repair):
170212024-02-15T03:05:11.826ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170222024-02-15T03:05:11.826Z 0 000-019 147 147 147 240 240 240 F F F
170232024-02-15T03:05:11.826Z 1 020-039 153 153 153 248 248 248 F F F
170242024-02-15T03:05:11.826Z 2 040-059 153 153 153 248 248 248 F F F
170252024-02-15T03:05:11.826Z 3 060-079 152 152 152 247 247 247 F F F
170262024-02-15T03:05:11.827Z 4 080-099 150 150 150 243 243 243 F F F
170272024-02-15T03:05:11.827Z 5 100-119 153 153 153 248 248 248 F F F
170282024-02-15T03:05:11.890Z 6 120-139 153 153 153 248 248 248 F F F
170292024-02-15T03:05:11.890Z 7 140-159 150 150 150 243 243 243 F F F
170302024-02-15T03:05:11.890Z 8 160-179 146 146 146 238 238 238 F F F
170312024-02-15T03:05:11.890Z 9 180-199 150 150 150 243 243 243 F F F
170322024-02-15T03:05:11.890Z 10 200-219 153 153 153 248 248 248 F F F
170332024-02-15T03:05:11.890Z 11 220-239 153 153 153 248 248 248 F F F
170342024-02-15T03:05:11.890Z 12 240-259 153 153 153 248 248 248 F F F
170352024-02-15T03:05:11.890Z 13 260-279 153 153 153 248 248 248 F F F
170362024-02-15T03:05:11.890Z 14 280-299 153 153 153 248 248 248 F F F
170372024-02-15T03:05:11.891Z 15 300-319 153 153 153 248 248 248 F F F
170382024-02-15T03:05:11.891Z 16 320-339 148 148 148 241 241 241 F F F
170392024-02-15T03:05:11.891Z 17 340-359 148 148 148 241 241 241 F F F
170402024-02-15T03:05:11.891Z 18 360-379 153 153 153 248 248 248 F F F
170412024-02-15T03:05:11.891Z 19 380-399 144 144 144 236 236 236 F F F
170422024-02-15T03:05:11.891Z 20 400-419 152 152 152 245 245 245 F F F
170432024-02-15T03:05:11.891Z 21 420-439 148 148 148 241 241 241 F F F
170442024-02-15T03:05:11.891Z 22 440-459 150 150 150 243 243 243 F F F
170452024-02-15T03:05:11.891Z 23 460-479 153 153 153 248 248 248 F F F
170462024-02-15T03:05:11.891Z 24 480-499 152 152 152 245 245 245 F F F
170472024-02-15T03:05:11.891Z 25 500-519 151 151 151 244 244 244 F F F
170482024-02-15T03:05:11.891Z 26 520-539 151 151 151 244 244 244 F F F
170492024-02-15T03:05:11.891Z 27 540-559 150 150 150 243 243 243 F F F
170502024-02-15T03:05:11.891Z 28 560-579 151 151 151 244 244 244 F F F
170512024-02-15T03:05:11.891Z 29 580-599 136 136 136 225 225 225 F F F
170522024-02-15T03:05:11.891ZMax gen: 153, Max flush: 248
170532024-02-15T03:05:11.891Z
170542024-02-15T03:05:11.891ZBegin loop 77
170552024-02-15T03:05:11.891ZDownstairs to restart: 0
170562024-02-15T03:05:11.892Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.711447817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760}
170572024-02-15T03:05:11.892Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.711685077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760}
170582024-02-15T03:05:11.892Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.711705205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760}
170592024-02-15T03:05:11.892Z{"msg":"Crucible 5138bdd2-5e1c-4373-bf54-6c9ceff2ef6e has session id: bf83671e-8a2d-45ae-b3c5-89ce91c627cb","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.711719153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170602024-02-15T03:05:11.892Z{"msg":"Upstairs opts: Upstairs UUID: 5138bdd2-5e1c-4373-bf54-6c9ceff2ef6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
170612024-02-15T03:05:11.892Z2024-02-15T03:05:10.711767847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170622024-02-15T03:05:11.892Z{"msg":"Crucible stats registered with UUID: 5138bdd2-5e1c-4373-bf54-6c9ceff2ef6e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.711807592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170632024-02-15T03:05:11.892Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.711864525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760}
170642024-02-15T03:05:11.892Z{"msg":"5138bdd2-5e1c-4373-bf54-6c9ceff2ef6e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712023895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170652024-02-15T03:05:11.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712051771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170662024-02-15T03:05:11.892Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712068509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170672024-02-15T03:05:11.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712083307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170682024-02-15T03:05:11.892Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712114183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170692024-02-15T03:05:11.892Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712156488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170702024-02-15T03:05:11.893Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712200232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170712024-02-15T03:05:11.893Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712228269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170722024-02-15T03:05:11.893Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712277753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170732024-02-15T03:05:11.893Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712321387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170742024-02-15T03:05:11.893Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712352193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170752024-02-15T03:05:11.893Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712397598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170762024-02-15T03:05:11.893Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:10.712456951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170772024-02-15T03:05:11.893Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:10.712501235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170782024-02-15T03:05:11.893Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:10.712596293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170792024-02-15T03:05:11.893Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712640037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170802024-02-15T03:05:11.893Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712663794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170812024-02-15T03:05:11.893Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.712731846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170822024-02-15T03:05:11.894Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.713117688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170832024-02-15T03:05:11.894Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.713154843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170842024-02-15T03:05:11.894Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.71317704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170852024-02-15T03:05:11.894Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.713214296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170862024-02-15T03:05:11.894Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.713776035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170872024-02-15T03:05:11.894Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.713839887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170882024-02-15T03:05:11.894Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.714352793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170892024-02-15T03:05:11.894Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.714405026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170902024-02-15T03:05:11.894Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.714433233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170912024-02-15T03:05:11.894Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.714519962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170922024-02-15T03:05:11.894Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.714564976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170932024-02-15T03:05:11.895Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:10.714592433Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170942024-02-15T03:05:12.827Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.714266947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170952024-02-15T03:05:12.827Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.714343147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170962024-02-15T03:05:12.846Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.715238625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170972024-02-15T03:05:12.846Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.71527624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170982024-02-15T03:05:12.846Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.715802964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
170992024-02-15T03:05:12.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716381292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171002024-02-15T03:05:12.846Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716410158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171012024-02-15T03:05:12.846Z{"msg":"[0]R flush_numbers[0..12]: [240, 248, 248, 247, 243, 248, 248, 243, 238, 243, 248, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716426506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171022024-02-15T03:05:12.846Z{"msg":"[0]R generation[0..12]: [147, 153, 153, 152, 150, 153, 153, 150, 146, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716441184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171032024-02-15T03:05:12.847Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716470161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171042024-02-15T03:05:12.847Z{"msg":"[1]R flush_numbers[0..12]: [240, 248, 248, 247, 243, 248, 248, 243, 238, 243, 248, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716493828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171052024-02-15T03:05:12.847Z{"msg":"[1]R generation[0..12]: [147, 153, 153, 152, 150, 153, 153, 150, 146, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716508666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171062024-02-15T03:05:12.847Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716530683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171072024-02-15T03:05:12.847Z{"msg":"[2]R flush_numbers[0..12]: [240, 248, 248, 247, 243, 248, 248, 243, 238, 243, 248, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716546231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171082024-02-15T03:05:12.847Z{"msg":"[2]R generation[0..12]: [147, 153, 153, 152, 150, 153, 153, 150, 146, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716567799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171092024-02-15T03:05:12.847Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716582737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171102024-02-15T03:05:12.847Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716608913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171112024-02-15T03:05:12.847Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716623781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171122024-02-15T03:05:12.847Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
171132024-02-15T03:05:12.847Z,"time":"2024-02-15T03:05:11.716644169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171142024-02-15T03:05:12.848Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716658657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"":"downstairs","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171152024-02-15T03:05:12.848Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716673435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171162024-02-15T03:05:12.848Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716693383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"0","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171172024-02-15T03:05:12.848Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716707861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"1","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171182024-02-15T03:05:12.848Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716729138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"client":"2","":"downstairs","session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171192024-02-15T03:05:12.848Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716743397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171202024-02-15T03:05:12.848Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716762984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171212024-02-15T03:05:12.848Z{"msg":"5138bdd2-5e1c-4373-bf54-6c9ceff2ef6e is now active with session: bf83671e-8a2d-45ae-b3c5-89ce91c627cb","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716777252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171222024-02-15T03:05:12.848Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.71679656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760,"session_id":"bf83671e-8a2d-45ae-b3c5-89ce91c627cb"}
171232024-02-15T03:05:12.849Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:11.716809418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760}
171242024-02-15T03:05:12.849ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171252024-02-15T03:05:12.849ZLoading write count information from file "/tmp/test_repair_verify.data"
171262024-02-15T03:05:12.849ZRun Repair workload
171272024-02-15T03:05:12.849Z01/30 Flush
171282024-02-15T03:05:12.849Z02/30 Flush
171292024-02-15T03:05:12.849Z03/30 Write block 271 len 24576 data: 11 13 12 9 10 11
171302024-02-15T03:05:12.849Z04/30 Write block 542 len 28672 data: 14 15 9 8 10 9 10
171312024-02-15T03:05:12.849Z05/30 Write block 595 len 4096 data: 12
171322024-02-15T03:05:12.850Z06/30 Write block 137 len 20480 data: 12 13 11 13 15
171332024-02-15T03:05:12.850Z07/30 Read block 505 len 28672
171342024-02-15T03:05:12.850Z08/30 Write block 39 len 40960 data: 13 11 12 13 13 15 17 13 15 13
171352024-02-15T03:05:12.850Z09/30 Write block 294 len 32768 data: 15 13 12 12 12 12 11 13
171362024-02-15T03:05:12.850Z10/30 Write block 18 len 20480 data: 10 9 7 6 6
171372024-02-15T03:05:12.850Z11/30 Read block 522 len 32768
171382024-02-15T03:05:12.850Z12/30 Write block 339 len 40960 data: 11 9 9 10 8 12 13 15 15 13
171392024-02-15T03:05:12.851Z13/30 Flush
171402024-02-15T03:05:12.864Z14/30 Write block 287 len 12288 data: 16 12 15
171412024-02-15T03:05:12.864Z15/30 Read block 432 len 32768
171422024-02-15T03:05:12.865Z16/30 Write block 36 len 16384 data: 15 14 15 14
171432024-02-15T03:05:12.865Z17/30 Write block 39 len 32768 data: 15 12 13 14 14 16 18 14
171442024-02-15T03:05:12.865Z18/30 Flush
171452024-02-15T03:05:12.865Z19/30 Write block 84 len 32768 data: 10 9 12 12 9 8 9 10
171462024-02-15T03:05:12.865Z20/30 Read block 558 len 12288
171472024-02-15T03:05:12.865Z21/30 Read block 157 len 8192
171482024-02-15T03:05:12.866Z22/30 Flush
171492024-02-15T03:05:12.866Z23/30 Write block 525 len 12288 data: 4 5 9
171502024-02-15T03:05:12.873Z24/30 Write block 209 len 32768 data: 11 9 10 12 11 13 12 11
171512024-02-15T03:05:12.873Z25/30 Write block 233 len 40960 data: 14 15 15 16 15 14 13 13 14 17
171522024-02-15T03:05:12.874Z26/30 Read block 574 len 28672
171532024-02-15T03:05:12.874Z27/30 Read block 485 len 32768
171542024-02-15T03:05:12.874Z28/30 Write block 470 len 32768 data: 18 14 11 9 8 8 8 9
171552024-02-15T03:05:12.874Z29/30 Read block 434 len 32768
171562024-02-15T03:05:12.874Z30/30 Write block 176 len 20480 data: 6 8 9 10 10
171572024-02-15T03:05:12.874Z----------------------------------------------------------------
171582024-02-15T03:05:12.875Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:17
171592024-02-15T03:05:12.875ZStates: Active Active Active
171602024-02-15T03:05:12.875ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
171612024-02-15T03:05:12.875Z 14 Acked 1013 Write 3 Done Done Done false
171622024-02-15T03:05:12.875Z 15 Acked 1014 Read 8 Sent Done Done false
171632024-02-15T03:05:12.875Z 16 Acked 1015 Write 4 Sent Done Done false
171642024-02-15T03:05:12.875Z 17 Acked 1016 Write 8 Sent Done Done false
171652024-02-15T03:05:12.876Z 18 Acked 1017 Flush 0 Sent Done Done false
171662024-02-15T03:05:12.876Z 19 Acked 1018 Write 8 Sent Done Done false
171672024-02-15T03:05:12.876Z 20 Acked 1019 Read 3 Sent Done Done false
171682024-02-15T03:05:12.876Z 21 Acked 1020 Read 2 Sent Done Done false
171692024-02-15T03:05:12.876Z 22 Acked 1021 Flush 0 Sent Done Done false
171702024-02-15T03:05:12.876Z 23 Acked 1022 Write 3 Sent Done Done false
171712024-02-15T03:05:12.876Z 24 Acked 1023 Write 8 Sent Done Done false
171722024-02-15T03:05:12.876Z 25 Acked 1024 Write 10 Sent Done Done false
171732024-02-15T03:05:12.876Z 26 Acked 1025 Read 7 Sent Done Done false
171742024-02-15T03:05:12.876Z 27 Acked 1026 Read 8 Sent Done Done false
171752024-02-15T03:05:12.876Z 28 Acked 1027 Write 8 Sent Done Done false
171762024-02-15T03:05:12.877Z 29 Acked 1028 Read 8 Sent Done Done false
171772024-02-15T03:05:12.877Z 30 Acked 1029 Write 5 Sent Sent Sent false
171782024-02-15T03:05:12.877Z STATES DS:0 DS:1 DS:2 TOTAL
171792024-02-15T03:05:12.877Z New 0 0 0 0
171802024-02-15T03:05:12.877Z Sent 16 1 1 18
171812024-02-15T03:05:12.877Z Done 1 16 16 33
171822024-02-15T03:05:12.877Z Skipped 0 0 0 0
171832024-02-15T03:05:12.877Z Error 0 0 0 0
171842024-02-15T03:05:12.877ZLast Flush: 1012 1021 1021
171852024-02-15T03:05:12.877ZDownstairs last five completed: 1012 1011 1010 1009 1008
171862024-02-15T03:05:12.877ZUpstairs last five completed: 30 29 28 27 26
171872024-02-15T03:05:12.877Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:11.754331227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1760}
171882024-02-15T03:05:12.877ZWrote out file "/tmp/test_repair_verify.data"
171892024-02-15T03:05:12.877Z
171902024-02-15T03:05:12.882ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171912024-02-15T03:05:12.891ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171922024-02-15T03:05:12.891Z 0 000-019 154 154 154 251 251 251 F F F
171932024-02-15T03:05:12.891Z 1 020-039 154 154 154 251 252 252 F F F <---
171942024-02-15T03:05:12.891Z 2 040-059 154 154 154 251 252 252 F F F <---
171952024-02-15T03:05:12.891Z 3 060-079 152 152 152 247 247 247 F F F
171962024-02-15T03:05:12.892Z 4 080-099 150 154 154 243 253 253 F F F <---
171972024-02-15T03:05:12.892Z 5 100-119 153 153 153 248 248 248 F F F
171982024-02-15T03:05:12.892Z 6 120-139 154 154 154 251 251 251 F F F
171992024-02-15T03:05:12.892Z 7 140-159 154 154 154 251 251 251 F F F
172002024-02-15T03:05:12.892Z 8 160-179 146 146 146 238 238 238 F T T <---
172012024-02-15T03:05:12.892Z 9 180-199 150 150 150 243 243 243 F T T <---
172022024-02-15T03:05:12.892Z 10 200-219 153 153 153 248 248 248 F T T <---
172032024-02-15T03:05:12.897Z 11 220-239 153 153 153 248 248 248 F T T <---
172042024-02-15T03:05:12.897Z 12 240-259 153 153 153 248 248 248 F T T <---
172052024-02-15T03:05:12.897Z 13 260-279 154 154 154 251 251 251 F F F
172062024-02-15T03:05:12.897Z 14 280-299 154 154 154 251 252 252 T F F <---
172072024-02-15T03:05:12.897Z 15 300-319 154 154 154 251 251 251 F F F
172082024-02-15T03:05:12.897Z 16 320-339 154 154 154 251 251 251 F F F
172092024-02-15T03:05:12.897Z 17 340-359 154 154 154 251 251 251 F F F
172102024-02-15T03:05:12.898Z 18 360-379 153 153 153 248 248 248 F F F
172112024-02-15T03:05:12.898Z 19 380-399 144 144 144 236 236 236 F F F
172122024-02-15T03:05:12.898Z 20 400-419 152 152 152 245 245 245 F F F
172132024-02-15T03:05:12.898Z 21 420-439 148 148 148 241 241 241 F F F
172142024-02-15T03:05:12.898Z 22 440-459 150 150 150 243 243 243 F F F
172152024-02-15T03:05:12.898Z 23 460-479 153 153 153 248 248 248 F T T <---
172162024-02-15T03:05:12.898Z 24 480-499 152 152 152 245 245 245 F F F
172172024-02-15T03:05:12.898Z 25 500-519 151 151 151 244 244 244 F F F
172182024-02-15T03:05:12.898Z 26 520-539 151 151 151 244 244 244 F T T <---
172192024-02-15T03:05:12.899Z 27 540-559 154 154 154 251 251 251 F F F
172202024-02-15T03:05:12.899Z 28 560-579 151 151 151 244 244 244 F F F
172212024-02-15T03:05:12.899Z 29 580-599 154 154 154 251 251 251 F F F
172222024-02-15T03:05:12.899ZMax gen: 154, Max flush: 253
172232024-02-15T03:05:12.899ZError: Difference in extent metadata found!
172242024-02-15T03:05:12.899ZA Difference in extent metadata is expected here
172252024-02-15T03:05:12.899ZOn loop 77
172262024-02-15T03:05:12.900Z
172272024-02-15T03:05:12.905ZVerifying data now
172282024-02-15T03:05:17.477ZNo change after verify
172292024-02-15T03:05:17.477ZLoop: 77 Downstairs dump after verify (and repair):
172302024-02-15T03:05:17.490ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172312024-02-15T03:05:17.490Z 0 000-019 154 154 154 251 251 251 F F F
172322024-02-15T03:05:17.490Z 1 020-039 154 154 154 252 252 252 F F F
172332024-02-15T03:05:17.490Z 2 040-059 154 154 154 252 252 252 F F F
172342024-02-15T03:05:17.490Z 3 060-079 152 152 152 247 247 247 F F F
172352024-02-15T03:05:17.490Z 4 080-099 154 154 154 253 253 253 F F F
172362024-02-15T03:05:17.494Z 5 100-119 153 153 153 248 248 248 F F F
172372024-02-15T03:05:17.495Z 6 120-139 154 154 154 251 251 251 F F F
172382024-02-15T03:05:17.495Z 7 140-159 154 154 154 251 251 251 F F F
172392024-02-15T03:05:17.495Z 8 160-179 155 155 155 254 254 254 F F F
172402024-02-15T03:05:17.495Z 9 180-199 155 155 155 254 254 254 F F F
172412024-02-15T03:05:17.495Z 10 200-219 155 155 155 254 254 254 F F F
172422024-02-15T03:05:17.495Z 11 220-239 155 155 155 254 254 254 F F F
172432024-02-15T03:05:17.495Z 12 240-259 155 155 155 254 254 254 F F F
172442024-02-15T03:05:17.495Z 13 260-279 154 154 154 251 251 251 F F F
172452024-02-15T03:05:17.495Z 14 280-299 154 154 154 252 252 252 F F F
172462024-02-15T03:05:17.495Z 15 300-319 154 154 154 251 251 251 F F F
172472024-02-15T03:05:17.496Z 16 320-339 154 154 154 251 251 251 F F F
172482024-02-15T03:05:17.496Z 17 340-359 154 154 154 251 251 251 F F F
172492024-02-15T03:05:17.496Z 18 360-379 153 153 153 248 248 248 F F F
172502024-02-15T03:05:17.496Z 19 380-399 144 144 144 236 236 236 F F F
172512024-02-15T03:05:17.496Z 20 400-419 152 152 152 245 245 245 F F F
172522024-02-15T03:05:17.496Z 21 420-439 148 148 148 241 241 241 F F F
172532024-02-15T03:05:17.496Z 22 440-459 150 150 150 243 243 243 F F F
172542024-02-15T03:05:17.497Z 23 460-479 155 155 155 254 254 254 F F F
172552024-02-15T03:05:17.497Z 24 480-499 152 152 152 245 245 245 F F F
172562024-02-15T03:05:17.497Z 25 500-519 151 151 151 244 244 244 F F F
172572024-02-15T03:05:17.497Z 26 520-539 155 155 155 254 254 254 F F F
172582024-02-15T03:05:17.497Z 27 540-559 154 154 154 251 251 251 F F F
172592024-02-15T03:05:17.497Z 28 560-579 151 151 151 244 244 244 F F F
172602024-02-15T03:05:17.497Z 29 580-599 154 154 154 251 251 251 F F F
172612024-02-15T03:05:17.497ZMax gen: 155, Max flush: 254
172622024-02-15T03:05:17.498Z
172632024-02-15T03:05:17.498ZBegin loop 78
172642024-02-15T03:05:17.498ZDownstairs to restart: 2
172652024-02-15T03:05:17.513Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403080842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768}
172662024-02-15T03:05:17.513Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403368746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768}
172672024-02-15T03:05:17.514Z{"msg":"Upstairs <-> Downstairs Message Version: 5"Crucible runtime is spawned
172682024-02-15T03:05:17.521Z,"v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.40341969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768}
172692024-02-15T03:05:17.521Z{"msg":"Crucible 029fae17-e929-4123-846c-d6403c9debb4 has session id: 3cbc5eef-81de-4691-9ce5-04082367cd8b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403461645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172702024-02-15T03:05:17.521Z{"msg":"Upstairs opts: Upstairs UUID: 029fae17-e929-4123-846c-d6403c9debb4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.40349674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172712024-02-15T03:05:17.521Z{"msg":"Crucible stats registered with UUID: 029fae17-e929-4123-846c-d6403c9debb4","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403544104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172722024-02-15T03:05:17.521Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.40358137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768}
172732024-02-15T03:05:17.521Z{"msg":"029fae17-e929-4123-846c-d6403c9debb4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403605097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172742024-02-15T03:05:17.521Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403645272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172752024-02-15T03:05:17.521Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403667949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172762024-02-15T03:05:17.522Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403697165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172772024-02-15T03:05:17.522Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403726421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172782024-02-15T03:05:17.522Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403766696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172792024-02-15T03:05:17.522Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403793693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172802024-02-15T03:05:17.522Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403830718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172812024-02-15T03:05:17.522Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403869543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172822024-02-15T03:05:17.522Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403912198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172832024-02-15T03:05:17.522Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403955103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172842024-02-15T03:05:17.522Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.403982789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172852024-02-15T03:05:17.522Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:16.404208811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172862024-02-15T03:05:17.522Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:16.404260305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172872024-02-15T03:05:17.522Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:16.404287541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172882024-02-15T03:05:17.522Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.404311528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172892024-02-15T03:05:17.522Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.404343624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172902024-02-15T03:05:17.522Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.404487296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172912024-02-15T03:05:17.522Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.40493761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172922024-02-15T03:05:17.522Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.404999012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172932024-02-15T03:05:17.522Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.405123417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172942024-02-15T03:05:17.522Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.405168591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172952024-02-15T03:05:17.522Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.405459174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172962024-02-15T03:05:17.522Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.405581949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172972024-02-15T03:05:17.522Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.405954492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172982024-02-15T03:05:17.522Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.405992497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
172992024-02-15T03:05:17.522Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.40605013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173002024-02-15T03:05:17.523Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.406128671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173012024-02-15T03:05:17.523Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.406184144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173022024-02-15T03:05:17.523Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:16.40621082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173032024-02-15T03:05:18.516Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.405989611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173042024-02-15T03:05:18.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.406097408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173052024-02-15T03:05:18.521Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.406888379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173062024-02-15T03:05:18.521Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.406949771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173072024-02-15T03:05:18.522Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.407416653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173082024-02-15T03:05:18.522Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.407963964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173092024-02-15T03:05:18.522Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408033495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173102024-02-15T03:05:18.522Z{"msg":"[0]R flush_numbers[0..12]: [251, 252, 252, 247, 253, 248, 251, 251, 254, 254, 254, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408058312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173112024-02-15T03:05:18.522Z{"msg":"[0]R generation[0..12]: [154, 154, 154, 152, 154, 153, 154, 154, 155, 155, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408084689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173122024-02-15T03:05:18.522Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408100697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173132024-02-15T03:05:18.522Z{"msg":"[1]R flush_numbers[0..12]: [251, 252, 252, 247, 253, 248, 251, 251, 254, 254, 254, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408123604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173142024-02-15T03:05:18.522Z{"msg":"[1]R generation[0..12]: [154, 154, 154, 152, 154, 153, 154, 154, 155, 155, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408138402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173152024-02-15T03:05:18.522Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408159679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173162024-02-15T03:05:18.522Z{"msg":"[2]R flush_numbers[0..12]: [251, 252, 252, 247, 253, 248, 251, 251, 254, 254, 254, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408174707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173172024-02-15T03:05:18.522Z{"msg":"[2]R generation[0..12]: [154, 154, 154, 152, 154, 153, 154, 154, 155, 155, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408189056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173182024-02-15T03:05:18.522Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408207373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173192024-02-15T03:05:18.522Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
173202024-02-15T03:05:18.522Z,"time":"2024-02-15T03:05:17.40823042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173212024-02-15T03:05:18.522Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408267526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173222024-02-15T03:05:18.522Z{"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408297312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173232024-02-15T03:05:18.522Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.40831231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"":"downstairs","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173242024-02-15T03:05:18.523Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408334877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173252024-02-15T03:05:18.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408348146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"0","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173262024-02-15T03:05:18.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408380592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"1","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173272024-02-15T03:05:18.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408417617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"client":"2","":"downstairs","session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173282024-02-15T03:05:18.523Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408442734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173292024-02-15T03:05:18.523Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.40847323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173302024-02-15T03:05:18.523Z{"msg":"029fae17-e929-4123-846c-d6403c9debb4 is now active with session: 3cbc5eef-81de-4691-9ce5-04082367cd8b","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408509396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173312024-02-15T03:05:18.523Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.408536332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768,"session_id":"3cbc5eef-81de-4691-9ce5-04082367cd8b"}
173322024-02-15T03:05:18.523Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:17.40855047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768}
173332024-02-15T03:05:18.523ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
173342024-02-15T03:05:18.523ZLoading write count information from file "/tmp/test_repair_verify.data"
173352024-02-15T03:05:18.523ZRun Repair workload
173362024-02-15T03:05:18.524Z01/30 Write block 473 len 8192 data: 10 9
173372024-02-15T03:05:18.527Z02/30 Read block 373 len 4096
173382024-02-15T03:05:18.527Z03/30 Write block 183 len 20480 data: 9 11 13 14 13
173392024-02-15T03:05:18.527Z04/30 Write block 270 len 40960 data: 12 12 14 13 10 11 12 13 9 11
173402024-02-15T03:05:18.527Z05/30 Read block 52 len 16384
173412024-02-15T03:05:18.528Z06/30 Flush
173422024-02-15T03:05:18.528Z07/30 Write block 10 len 32768 data: 14 10 8 7 8 7 7 8
173432024-02-15T03:05:18.531Z08/30 Write block 442 len 20480 data: 14 13 11 13 14
173442024-02-15T03:05:18.531Z09/30 Read block 178 len 4096
173452024-02-15T03:05:18.531Z10/30 Read block 403 len 40960
173462024-02-15T03:05:18.531Z11/30 Flush
173472024-02-15T03:05:18.534Z12/30 Write block 350 len 32768 data: 16 17 16 16 15 13 12 12
173482024-02-15T03:05:18.534Z13/30 Read block 48 len 8192
173492024-02-15T03:05:18.535Z14/30 Flush
173502024-02-15T03:05:18.538Z15/30 Write block 566 len 36864 data: 10 10 9 8 10 11 11 14 15
173512024-02-15T03:05:18.538Z16/30 Read block 315 len 24576
173522024-02-15T03:05:18.563Z17/30 Read block 438 len 20480
173532024-02-15T03:05:18.564Z18/30 Write block 338 len 4096 data: 15
173542024-02-15T03:05:18.564Z19/30 Read block 5 len 32768
173552024-02-15T03:05:18.564Z20/30 Read block 467 len 36864
173562024-02-15T03:05:18.564Z21/30 Read block 489 len 28672
173572024-02-15T03:05:18.569Z22/30 Write block 481 len 40960 data: 14 13 9 11 9 12 13 9 12 14
173582024-02-15T03:05:18.569Z23/30 Write block 108 len 28672 data: 8 8 8 10 8 7 8
173592024-02-15T03:05:18.569Z24/30 Write block 182 len 16384 data: 6 10 12 14
173602024-02-15T03:05:18.569Z25/30 Write block 349 len 12288 data: 14 17 18
173612024-02-15T03:05:18.569Z26/30 Write block 72 len 36864 data: 12 7 7 10 10 8 7 7 6
173622024-02-15T03:05:18.569Z27/30 Read block 39 len 20480
173632024-02-15T03:05:18.569Z28/30 Read block 20 len 28672
173642024-02-15T03:05:18.569Z29/30 Write block 448 len 40960 data: 12 9 8 8 7 7 8 10 9 8
173652024-02-15T03:05:18.569Z30/30 Read block 43 len 28672
173662024-02-15T03:05:18.569Z----------------------------------------------------------------
173672024-02-15T03:05:18.570Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
173682024-02-15T03:05:18.570ZStates: Active Active Active
173692024-02-15T03:05:18.570ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
173702024-02-15T03:05:18.570Z 1 Acked 1000 Write 2 Done Done Done false
173712024-02-15T03:05:18.570Z 2 Acked 1001 Read 1 Done Done Done false
173722024-02-15T03:05:18.570Z 3 Acked 1002 Write 5 Done Done Sent false
173732024-02-15T03:05:18.570Z 4 Acked 1003 Write 10 Done Done Sent false
173742024-02-15T03:05:18.570Z 5 Acked 1004 Read 4 Done Done Sent false
173752024-02-15T03:05:18.570Z 6 Acked 1005 Flush 0 Done Done Sent false
173762024-02-15T03:05:18.570Z 7 Acked 1006 Write 8 Done Done Sent false
173772024-02-15T03:05:18.570Z 8 Acked 1007 Write 5 Done Done Sent false
173782024-02-15T03:05:18.570Z 9 Acked 1008 Read 1 Done Done Sent false
173792024-02-15T03:05:18.571Z 10 Acked 1009 Read 10 Done Done Sent false
173802024-02-15T03:05:18.571Z 11 Acked 1010 Flush 0 Done Done Sent false
173812024-02-15T03:05:18.571Z 12 Acked 1011 Write 8 Done Done Sent false
173822024-02-15T03:05:18.571Z 13 Acked 1012 Read 2 Done Done Sent false
173832024-02-15T03:05:18.571Z 14 Acked 1013 Flush 0 Done Done Sent false
173842024-02-15T03:05:18.571Z 15 Acked 1014 Write 9 Done Done Sent false
173852024-02-15T03:05:18.571Z 16 Acked 1015 Read 6 Done Done Sent false
173862024-02-15T03:05:18.571Z 17 Acked 1016 Read 5 Done Done Sent false
173872024-02-15T03:05:18.571Z 18 Acked 1017 Write 1 Done Done Sent false
173882024-02-15T03:05:18.571Z 19 Acked 1018 Read 8 Done Done Sent false
173892024-02-15T03:05:18.571Z 20 Acked 1019 Read 9 Done Done Sent false
173902024-02-15T03:05:18.571Z 21 Acked 1020 Read 7 Done Done Sent false
173912024-02-15T03:05:18.572Z 22 Acked 1021 Write 10 Done Done Sent false
173922024-02-15T03:05:18.572Z 23 Acked 1022 Write 7 Done Done Sent false
173932024-02-15T03:05:18.572Z 24 Acked 1023 Write 4 Done Done Sent false
173942024-02-15T03:05:18.572Z 25 Acked 1024 Write 3 Done Done Sent false
173952024-02-15T03:05:18.572Z 26 Acked 1025 Write 9 Done Done Sent false
173962024-02-15T03:05:18.572Z 27 Acked 1026 Read 5 Done Done Sent false
173972024-02-15T03:05:18.572Z 28 Acked 1027 Read 7 Done Done Sent false
173982024-02-15T03:05:18.572Z 29 Acked 1028 Write 10 Done Done Sent false
173992024-02-15T03:05:18.572Z 30 Acked 1029 Read 7 Done Sent Sent false
174002024-02-15T03:05:18.572Z STATES DS:0 DS:1 DS:2 TOTAL
174012024-02-15T03:05:18.572Z New 0 0 0 0
174022024-02-15T03:05:18.572Z Sent 0 1 28 29
174032024-02-15T03:05:18.572Z Done 30 29 2 61
174042024-02-15T03:05:18.573Z Skipped 0 0 0 0
174052024-02-15T03:05:18.573Z Error 0 0 0 0
174062024-02-15T03:05:18.573ZLast Flush: 1013 1013 0
174072024-02-15T03:05:18.573ZDownstairs last five completed:
174082024-02-15T03:05:18.573ZUpstairs last five completed: 30 29 28 27 26
174092024-02-15T03:05:18.573Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:17.442342826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1768}
174102024-02-15T03:05:18.573ZWrote out file "/tmp/test_repair_verify.data"
174112024-02-15T03:05:18.573Z
174122024-02-15T03:05:18.573ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174132024-02-15T03:05:18.580ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174142024-02-15T03:05:18.580Z 0 000-019 156 156 154 256 256 251 F F F <---
174152024-02-15T03:05:18.580Z 1 020-039 154 154 154 252 252 252 F F F
174162024-02-15T03:05:18.580Z 2 040-059 154 154 154 252 252 252 F F F
174172024-02-15T03:05:18.580Z 3 060-079 152 152 152 247 247 247 T T F <---
174182024-02-15T03:05:18.581Z 4 080-099 154 154 154 253 253 253 T T F <---
174192024-02-15T03:05:18.581Z 5 100-119 153 153 153 248 248 248 T T F <---
174202024-02-15T03:05:18.581Z 6 120-139 154 154 154 251 251 251 F F F
174212024-02-15T03:05:18.581Z 7 140-159 154 154 154 251 251 251 F F F
174222024-02-15T03:05:18.581Z 8 160-179 155 155 155 254 254 254 F F F
174232024-02-15T03:05:18.581Z 9 180-199 156 156 155 255 255 254 T T F <---
174242024-02-15T03:05:18.581Z 10 200-219 155 155 155 254 254 254 F F F
174252024-02-15T03:05:18.581Z 11 220-239 155 155 155 254 254 254 F F F
174262024-02-15T03:05:18.581Z 12 240-259 155 155 155 254 254 254 F F F
174272024-02-15T03:05:18.585Z 13 260-279 156 156 154 255 255 251 F F F <---
174282024-02-15T03:05:18.585Z 14 280-299 154 154 154 252 252 252 F F F
174292024-02-15T03:05:18.585Z 15 300-319 154 154 154 251 251 251 F F F
174302024-02-15T03:05:18.585Z 16 320-339 154 154 154 251 251 251 T T F <---
174312024-02-15T03:05:18.585Z 17 340-359 156 156 154 257 257 251 T T F <---
174322024-02-15T03:05:18.586Z 18 360-379 153 153 153 248 248 248 F F F
174332024-02-15T03:05:18.586Z 19 380-399 144 144 144 236 236 236 F F F
174342024-02-15T03:05:18.586Z 20 400-419 152 152 152 245 245 245 F F F
174352024-02-15T03:05:18.586Z 21 420-439 148 148 148 241 241 241 F F F
174362024-02-15T03:05:18.586Z 22 440-459 156 156 150 256 256 243 T T F <---
174372024-02-15T03:05:18.586Z 23 460-479 156 156 155 255 255 254 F F T <---
174382024-02-15T03:05:18.586Z 24 480-499 152 152 152 245 245 245 T T F <---
174392024-02-15T03:05:18.586Z 25 500-519 151 151 151 244 244 244 F F F
174402024-02-15T03:05:18.586Z 26 520-539 155 155 155 254 254 254 F F F
174412024-02-15T03:05:18.586Z 27 540-559 154 154 154 251 251 251 F F F
174422024-02-15T03:05:18.586Z 28 560-579 151 151 151 244 244 244 T T F <---
174432024-02-15T03:05:18.586Z 29 580-599 154 154 154 251 251 251 F F F
174442024-02-15T03:05:18.586ZMax gen: 156, Max flush: 257
174452024-02-15T03:05:18.586ZError: Difference in extent metadata found!
174462024-02-15T03:05:18.586ZA Difference in extent metadata is expected here
174472024-02-15T03:05:18.586ZOn loop 78
174482024-02-15T03:05:18.587Z
174492024-02-15T03:05:18.590ZVerifying data now
174502024-02-15T03:05:24.209ZNo change after verify
174512024-02-15T03:05:24.209ZLoop: 78 Downstairs dump after verify (and repair):
174522024-02-15T03:05:24.230ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174532024-02-15T03:05:24.230Z 0 000-019 156 156 156 256 256 256 F F F
174542024-02-15T03:05:24.230Z 1 020-039 154 154 154 252 252 252 F F F
174552024-02-15T03:05:24.231Z 2 040-059 154 154 154 252 252 252 F F F
174562024-02-15T03:05:24.231Z 3 060-079 157 157 157 258 258 258 F F F
174572024-02-15T03:05:24.231Z 4 080-099 157 157 157 258 258 258 F F F
174582024-02-15T03:05:24.231Z 5 100-119 157 157 157 258 258 258 F F F
174592024-02-15T03:05:24.231Z 6 120-139 154 154 154 251 251 251 F F F
174602024-02-15T03:05:24.231Z 7 140-159 154 154 154 251 251 251 F F F
174612024-02-15T03:05:24.231Z 8 160-179 155 155 155 254 254 254 F F F
174622024-02-15T03:05:24.231Z 9 180-199 157 157 157 258 258 258 F F F
174632024-02-15T03:05:24.231Z 10 200-219 155 155 155 254 254 254 F F F
174642024-02-15T03:05:24.239Z 11 220-239 155 155 155 254 254 254 F F F
174652024-02-15T03:05:24.239Z 12 240-259 155 155 155 254 254 254 F F F
174662024-02-15T03:05:24.239Z 13 260-279 156 156 156 255 255 255 F F F
174672024-02-15T03:05:24.239Z 14 280-299 154 154 154 252 252 252 F F F
174682024-02-15T03:05:24.239Z 15 300-319 154 154 154 251 251 251 F F F
174692024-02-15T03:05:24.239Z 16 320-339 157 157 157 258 258 258 F F F
174702024-02-15T03:05:24.240Z 17 340-359 157 157 157 258 258 258 F F F
174712024-02-15T03:05:24.240Z 18 360-379 153 153 153 248 248 248 F F F
174722024-02-15T03:05:24.240Z 19 380-399 144 144 144 236 236 236 F F F
174732024-02-15T03:05:24.240Z 20 400-419 152 152 152 245 245 245 F F F
174742024-02-15T03:05:24.240Z 21 420-439 148 148 148 241 241 241 F F F
174752024-02-15T03:05:24.240Z 22 440-459 157 157 157 258 258 258 F F F
174762024-02-15T03:05:24.240Z 23 460-479 156 156 156 255 255 255 F F F
174772024-02-15T03:05:24.241Z 24 480-499 157 157 157 258 258 258 F F F
174782024-02-15T03:05:24.241Z 25 500-519 151 151 151 244 244 244 F F F
174792024-02-15T03:05:24.241Z 26 520-539 155 155 155 254 254 254 F F F
174802024-02-15T03:05:24.241Z 27 540-559 154 154 154 251 251 251 F F F
174812024-02-15T03:05:24.241Z 28 560-579 157 157 157 258 258 258 F F F
174822024-02-15T03:05:24.241Z 29 580-599 154 154 154 251 251 251 F F F
174832024-02-15T03:05:24.241ZMax gen: 157, Max flush: 258
174842024-02-15T03:05:24.242Z
174852024-02-15T03:05:24.242ZBegin loop 79
174862024-02-15T03:05:24.242ZDownstairs to restart: 0
174872024-02-15T03:05:24.259Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139111488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776}
174882024-02-15T03:05:24.259Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","levelCrucible runtime is spawned
174892024-02-15T03:05:24.259Z":30,"time":"2024-02-15T03:05:23.139434748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776}
174902024-02-15T03:05:24.720Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139468643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776}
174912024-02-15T03:05:24.720Z{"msg":"Crucible 0e2826e4-3820-4321-8ff7-bdd6970754a0 has session id: b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139508598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
174922024-02-15T03:05:24.720Z{"msg":"Upstairs opts: Upstairs UUID: 0e2826e4-3820-4321-8ff7-bdd6970754a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139557512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
174932024-02-15T03:05:24.720Z{"msg":"Crucible stats registered with UUID: 0e2826e4-3820-4321-8ff7-bdd6970754a0","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139589418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
174942024-02-15T03:05:24.721Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139613325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776}
174952024-02-15T03:05:24.721Z{"msg":"0e2826e4-3820-4321-8ff7-bdd6970754a0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139643122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
174962024-02-15T03:05:24.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139674607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
174972024-02-15T03:05:24.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139698615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
174982024-02-15T03:05:24.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.13973358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
174992024-02-15T03:05:24.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139757557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175002024-02-15T03:05:24.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139782324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175012024-02-15T03:05:24.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139800582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175022024-02-15T03:05:24.722Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139833618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175032024-02-15T03:05:24.722Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139866874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175042024-02-15T03:05:24.722Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.139910098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175052024-02-15T03:05:24.722Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:23.139941664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175062024-02-15T03:05:24.722Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:23.139980389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175072024-02-15T03:05:24.722Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.140015695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175082024-02-15T03:05:24.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.140058079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175092024-02-15T03:05:24.723Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:23.140096195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175102024-02-15T03:05:24.723Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.14013542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175112024-02-15T03:05:24.723Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.140174085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175122024-02-15T03:05:24.723Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.14029764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175132024-02-15T03:05:24.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.140747163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175142024-02-15T03:05:24.723Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.140788128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175152024-02-15T03:05:24.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.140822544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175162024-02-15T03:05:24.723Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.14085579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175172024-02-15T03:05:24.723Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.141377644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175182024-02-15T03:05:24.723Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.141527685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175192024-02-15T03:05:24.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.141834457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175202024-02-15T03:05:24.724Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.141872402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175212024-02-15T03:05:24.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.14188915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175222024-02-15T03:05:24.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.142090045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175232024-02-15T03:05:24.724Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.14212568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175242024-02-15T03:05:24.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:23.142142038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175252024-02-15T03:05:25.264Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.142019057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175262024-02-15T03:05:25.264Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.142244839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175272024-02-15T03:05:25.276Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.143160634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175282024-02-15T03:05:25.276Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.143199799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175292024-02-15T03:05:25.276Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.143700356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175302024-02-15T03:05:25.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144318129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175312024-02-15T03:05:25.276Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144422546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175322024-02-15T03:05:25.276Z{"msg":"[0]R flush_numbers[0..12]: [256, 252, 252, 258, 258, 258, 251, 251, 254, 258, 254, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144487478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175332024-02-15T03:05:25.276Z{"msg":"[0]R generation[0..12]: [156, 154, 154, 157, 157, 157, 154, 154, 155, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144534692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175342024-02-15T03:05:25.276Z{"msg":"Wait for a query_work_queue command to finish before sending IO
175352024-02-15T03:05:25.276Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144583286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175362024-02-15T03:05:25.276Z{"msg":"[1]R flush_numbers[0..12]: [256, 252, 252, 258, 258, 258, 251, 251, 254, 258, 254, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.14462632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175372024-02-15T03:05:25.276Z{"msg":"[1]R generation[0..12]: [156, 154, 154, 157, 157, 157, 154, 154, 155, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144667935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175382024-02-15T03:05:25.276Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.14470684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175392024-02-15T03:05:25.276Z{"msg":"[2]R flush_numbers[0..12]: [256, 252, 252, 258, 258, 258, 251, 251, 254, 258, 254, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144745335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175402024-02-15T03:05:25.276Z{"msg":"[2]R generation[0..12]: [156, 154, 154, 157, 157, 157, 154, 154, 155, 157, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144783201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175412024-02-15T03:05:25.276Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144823205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175422024-02-15T03:05:25.276Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144860711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175432024-02-15T03:05:25.276Z{"msg":"Generation requested: 158 >= found:158Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
175442024-02-15T03:05:25.276Z","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144901776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175452024-02-15T03:05:25.276Z{"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144938811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175462024-02-15T03:05:25.276Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144964178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"":"downstairs","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175472024-02-15T03:05:25.276Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.144993734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175482024-02-15T03:05:25.276Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.145007113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"0","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175492024-02-15T03:05:25.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.14502759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"1","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175502024-02-15T03:05:25.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.145041748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"client":"2","":"downstairs","session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175512024-02-15T03:05:25.277Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.145061666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175522024-02-15T03:05:25.277Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.145074484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175532024-02-15T03:05:25.277Z{"msg":"0e2826e4-3820-4321-8ff7-bdd6970754a0 is now active with session: b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.145093911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175542024-02-15T03:05:25.277Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.14510607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776,"session_id":"b8411f33-ba2f-4a7f-8d1a-6a2e07a370d8"}
175552024-02-15T03:05:25.277Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:24.145125078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776}
175562024-02-15T03:05:25.277ZLoading write count information from file "/tmp/test_repair_verify.data"
175572024-02-15T03:05:25.277ZRun Repair workload
175582024-02-15T03:05:25.277Z01/30 Write block 520 len 16384 data: 8 6 5 6
175592024-02-15T03:05:25.277Z02/30 Write block 38 len 40960 data: 16 16 13 14 15 15 17 19 15 16
175602024-02-15T03:05:25.277Z03/30 Write block 290 len 28672 data: 22 20 18 16 16 14 13
175612024-02-15T03:05:25.277Z04/30 Write block 215 len 36864 data: 13 12 11 11 13 14 14 13 11
175622024-02-15T03:05:25.277Z05/30 Read block 365 len 36864
175632024-02-15T03:05:25.277Z06/30 Write block 553 len 36864 data: 10 12 11 9 14 15 18 17 13
175642024-02-15T03:05:25.277Z07/30 Read block 21 len 36864
175652024-02-15T03:05:25.277Z08/30 Write block 124 len 28672 data: 8 8 10 10 10 10 10
175662024-02-15T03:05:25.277Z09/30 Write block 188 len 28672 data: 14 17 15 14 14 17 17
175672024-02-15T03:05:25.277Z10/30 Read block 585 len 32768
175682024-02-15T03:05:25.277Z11/30 Write block 125 len 40960 data: 9 11 11 11 11 11 11 12 13 12
175692024-02-15T03:05:25.277Z12/30 Write block 150 len 4096 data: 10
175702024-02-15T03:05:25.277Z13/30 Read block 375 len 32768
175712024-02-15T03:05:25.284Z14/30 Read block 397 len 16384
175722024-02-15T03:05:25.284Z15/30 Write block 4 len 40960 data: 10 10 11 10 13 11 15 11 9 8
175732024-02-15T03:05:25.284Z16/30 Write block 305 len 24576 data: 17 14 13 13 13 14
175742024-02-15T03:05:25.284Z17/30 Read block 525 len 20480
175752024-02-15T03:05:25.284Z18/30 Read block 239 len 32768
175762024-02-15T03:05:25.285Z19/30 Write block 125 len 32768 data: 10 12 12 12 12 12 12 13
175772024-02-15T03:05:25.285Z20/30 Read block 329 len 32768
175782024-02-15T03:05:25.285Z21/30 Write block 396 len 32768 data: 12 12 12 14 12 11 10 9
175792024-02-15T03:05:25.285Z22/30 Flush
175802024-02-15T03:05:25.293Z23/30 Write block 352 len 4096 data: 17
175812024-02-15T03:05:25.293Z24/30 Read block 329 len 12288
175822024-02-15T03:05:25.293Z25/30 Write block 262 len 20480 data: 7 7 7 10 10
175832024-02-15T03:05:25.293Z26/30 Read block 492 len 36864
175842024-02-15T03:05:25.293Z27/30 Write block 547 len 36864 data: 10 11 9 8 7 7 11 13 12
175852024-02-15T03:05:25.293Z28/30 Read block 296 len 28672
175862024-02-15T03:05:25.301Z29/30 Read block 122 len 16384
175872024-02-15T03:05:25.301Z30/30 Write block 509 len 12288 data: 13 12 11
175882024-02-15T03:05:25.301Z----------------------------------------------------------------
175892024-02-15T03:05:25.301Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
175902024-02-15T03:05:25.301ZStates: Active Active Active
175912024-02-15T03:05:25.301ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
175922024-02-15T03:05:25.301Z 1 Acked 1000 Write 4 Done Done Done false
175932024-02-15T03:05:25.301Z 2 Acked 1001 Write 10 Done Done Done false
175942024-02-15T03:05:25.301Z 3 Acked 1002 Write 7 Sent Done Done false
175952024-02-15T03:05:25.301Z 4 Acked 1003 Write 9 Sent Done Done false
175962024-02-15T03:05:25.301Z 5 Acked 1004 Read 9 Sent Done Done false
175972024-02-15T03:05:25.301Z 6 Acked 1005 Write 9 Sent Done Done false
175982024-02-15T03:05:25.301Z 7 Acked 1006 Read 9 Sent Done Done false
175992024-02-15T03:05:25.301Z 8 Acked 1007 Write 7 Sent Done Done false
176002024-02-15T03:05:25.301Z 9 Acked 1008 Write 7 Sent Done Done false
176012024-02-15T03:05:25.301Z 10 Acked 1009 Read 8 Sent Done Done false
176022024-02-15T03:05:25.301Z 11 Acked 1010 Write 10 Sent Done Done false
176032024-02-15T03:05:25.301Z 12 Acked 1011 Write 1 Sent Done Done false
176042024-02-15T03:05:25.301Z 13 Acked 1012 Read 8 Sent Done Done false
176052024-02-15T03:05:25.301Z 14 Acked 1013 Read 4 Sent Done Done false
176062024-02-15T03:05:25.301Z 15 Acked 1014 Write 10 Sent Done Done false
176072024-02-15T03:05:25.302Z 16 Acked 1015 Write 6 Sent Done Done false
176082024-02-15T03:05:25.302Z 17 Acked 1016 Read 5 Sent Done Done false
176092024-02-15T03:05:25.302Z 18 Acked 1017 Read 8 Sent Done Done false
176102024-02-15T03:05:25.302Z 19 Acked 1018 Write 8 Sent Done Done false
176112024-02-15T03:05:25.302Z 20 Acked 1019 Read 8 Sent Done Done false
176122024-02-15T03:05:25.302Z 21 Acked 1020 Write 8 Sent Done Done false
176132024-02-15T03:05:25.302Z 22 Acked 1021 Flush 0 Sent Done Done false
176142024-02-15T03:05:25.302Z 23 Acked 1022 Write 1 Sent Done Done false
176152024-02-15T03:05:25.302Z 24 Acked 1023 Read 3 Sent Done Done false
176162024-02-15T03:05:25.302Z 25 Acked 1024 Write 5 Sent Done Done false
176172024-02-15T03:05:25.302Z 26 Acked 1025 Read 9 Sent Done Done false
176182024-02-15T03:05:25.302Z 27 Acked 1026 Write 9 Sent Done Done false
176192024-02-15T03:05:25.302Z 28 Acked 1027 Read 7 Sent Done Done false
176202024-02-15T03:05:25.302Z 29 Acked 1028 Read 4 Sent Done Done false
176212024-02-15T03:05:25.302Z 30 Acked 1029 Write 3 Sent Sent Sent false
176222024-02-15T03:05:25.302Z STATES DS:0 DS:1 DS:2 TOTAL
176232024-02-15T03:05:25.302Z New 0 0 0 0
176242024-02-15T03:05:25.302Z Sent 28 1 1 30
176252024-02-15T03:05:25.302Z Done 2 29 29 60
176262024-02-15T03:05:25.302Z Skipped 0 0 0 0
176272024-02-15T03:05:25.302Z Error 0 0 0 0
176282024-02-15T03:05:25.302ZLast Flush: 0 1021 1021
176292024-02-15T03:05:25.302ZDownstairs last five completed:
176302024-02-15T03:05:25.302ZUpstairs last five completed: 30 29 28 27 26
176312024-02-15T03:05:25.303Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:24.181587099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1776}
176322024-02-15T03:05:25.303ZWrote out file "/tmp/test_repair_verify.data"
176332024-02-15T03:05:25.303Z
176342024-02-15T03:05:25.309ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176352024-02-15T03:05:25.320ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176362024-02-15T03:05:25.321Z 0 000-019 156 158 158 256 259 259 F F F <---
176372024-02-15T03:05:25.321Z 1 020-039 154 158 158 252 259 259 T F F <---
176382024-02-15T03:05:25.321Z 2 040-059 154 158 158 252 259 259 T F F <---
176392024-02-15T03:05:25.321Z 3 060-079 157 157 157 258 258 258 F F F
176402024-02-15T03:05:25.321Z 4 080-099 157 157 157 258 258 258 F F F
176412024-02-15T03:05:25.330Z 5 100-119 157 157 157 258 258 258 F F F
176422024-02-15T03:05:25.330Z 6 120-139 154 158 158 251 259 259 F F F <---
176432024-02-15T03:05:25.330Z 7 140-159 154 158 158 251 259 259 F F F <---
176442024-02-15T03:05:25.330Z 8 160-179 155 155 155 254 254 254 F F F
176452024-02-15T03:05:25.331Z 9 180-199 157 158 158 258 259 259 F F F <---
176462024-02-15T03:05:25.331Z 10 200-219 155 158 158 254 259 259 F F F <---
176472024-02-15T03:05:25.331Z 11 220-239 155 158 158 254 259 259 F F F <---
176482024-02-15T03:05:25.332Z 12 240-259 155 155 155 254 254 254 F F F
176492024-02-15T03:05:25.332Z 13 260-279 156 156 156 255 255 255 F T T <---
176502024-02-15T03:05:25.332Z 14 280-299 154 158 158 252 259 259 F F F <---
176512024-02-15T03:05:25.332Z 15 300-319 154 158 158 251 259 259 F F F <---
176522024-02-15T03:05:25.332Z 16 320-339 157 157 157 258 258 258 F F F
176532024-02-15T03:05:25.332Z 17 340-359 157 157 157 258 258 258 F T T <---
176542024-02-15T03:05:25.332Z 18 360-379 153 153 153 248 248 248 F F F
176552024-02-15T03:05:25.332Z 19 380-399 144 158 158 236 259 259 F F F <---
176562024-02-15T03:05:25.332Z 20 400-419 152 158 158 245 259 259 F F F <---
176572024-02-15T03:05:25.332Z 21 420-439 148 148 148 241 241 241 F F F
176582024-02-15T03:05:25.332Z 22 440-459 157 157 157 258 258 258 F F F
176592024-02-15T03:05:25.332Z 23 460-479 156 156 156 255 255 255 F F F
176602024-02-15T03:05:25.333Z 24 480-499 157 157 157 258 258 258 F F F
176612024-02-15T03:05:25.333Z 25 500-519 151 151 151 244 244 244 F T T <---
176622024-02-15T03:05:25.333Z 26 520-539 155 158 158 254 259 259 T F F <---
176632024-02-15T03:05:25.333Z 27 540-559 154 158 158 251 259 259 F T T <---
176642024-02-15T03:05:25.333Z 28 560-579 157 158 158 258 259 259 F F F <---
176652024-02-15T03:05:25.333Z 29 580-599 154 154 154 251 251 251 F F F
176662024-02-15T03:05:25.333ZMax gen: 158, Max flush: 259
176672024-02-15T03:05:25.333ZError: Difference in extent metadata found!
176682024-02-15T03:05:25.333ZA Difference in extent metadata is expected here
176692024-02-15T03:05:25.333ZOn loop 79
176702024-02-15T03:05:25.333Z
176712024-02-15T03:05:25.333ZVerifying data now
176722024-02-15T03:05:31.199ZNo change after verify
176732024-02-15T03:05:31.199ZLoop: 79 Downstairs dump after verify (and repair):
176742024-02-15T03:05:31.217ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176752024-02-15T03:05:31.217Z 0 000-019 158 158 158 259 259 259 F F F
176762024-02-15T03:05:31.217Z 1 020-039 158 158 158 259 259 259 F F F
176772024-02-15T03:05:31.217Z 2 040-059 158 158 158 259 259 259 F F F
176782024-02-15T03:05:31.217Z 3 060-079 157 157 157 258 258 258 F F F
176792024-02-15T03:05:31.217Z 4 080-099 157 157 157 258 258 258 F F F
176802024-02-15T03:05:31.217Z 5 100-119 157 157 157 258 258 258 F F F
176812024-02-15T03:05:31.217Z 6 120-139 158 158 158 259 259 259 F F F
176822024-02-15T03:05:31.220Z 7 140-159 158 158 158 259 259 259 F F F
176832024-02-15T03:05:31.221Z 8 160-179 155 155 155 254 254 254 F F F
176842024-02-15T03:05:31.221Z 9 180-199 158 158 158 259 259 259 F F F
176852024-02-15T03:05:31.221Z 10 200-219 158 158 158 259 259 259 F F F
176862024-02-15T03:05:31.221Z 11 220-239 158 158 158 259 259 259 F F F
176872024-02-15T03:05:31.222Z 12 240-259 155 155 155 254 254 254 F F F
176882024-02-15T03:05:31.222Z 13 260-279 159 159 159 260 260 260 F F F
176892024-02-15T03:05:31.222Z 14 280-299 158 158 158 259 259 259 F F F
176902024-02-15T03:05:31.222Z 15 300-319 158 158 158 259 259 259 F F F
176912024-02-15T03:05:31.222Z 16 320-339 157 157 157 258 258 258 F F F
176922024-02-15T03:05:31.222Z 17 340-359 159 159 159 260 260 260 F F F
176932024-02-15T03:05:31.223Z 18 360-379 153 153 153 248 248 248 F F F
176942024-02-15T03:05:31.223Z 19 380-399 158 158 158 259 259 259 F F F
176952024-02-15T03:05:31.223Z 20 400-419 158 158 158 259 259 259 F F F
176962024-02-15T03:05:31.223Z 21 420-439 148 148 148 241 241 241 F F F
176972024-02-15T03:05:31.223Z 22 440-459 157 157 157 258 258 258 F F F
176982024-02-15T03:05:31.223Z 23 460-479 156 156 156 255 255 255 F F F
176992024-02-15T03:05:31.223Z 24 480-499 157 157 157 258 258 258 F F F
177002024-02-15T03:05:31.223Z 25 500-519 159 159 159 260 260 260 F F F
177012024-02-15T03:05:31.223Z 26 520-539 158 158 158 259 259 259 F F F
177022024-02-15T03:05:31.223Z 27 540-559 159 159 159 260 260 260 F F F
177032024-02-15T03:05:31.223Z 28 560-579 158 158 158 259 259 259 F F F
177042024-02-15T03:05:31.223Z 29 580-599 154 154 154 251 251 251 F F F
177052024-02-15T03:05:31.224ZMax gen: 159, Max flush: 260
177062024-02-15T03:05:31.224Z
177072024-02-15T03:05:31.224ZBegin loop 80
177082024-02-15T03:05:31.224ZDownstairs to restart: 1
177092024-02-15T03:05:31.238Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.126617025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784}
177102024-02-15T03:05:31.239Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.126943524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784}
177112024-02-15T03:05:31.239Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
177122024-02-15T03:05:31.245Z,"time":"2024-02-15T03:05:30.12697291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784}
177132024-02-15T03:05:31.245Z{"msg":"Crucible 26117b2f-933c-4ebf-bd08-528a56f2add9 has session id: daa67e2b-6ff3-4e5a-bbc1-2f060ea66818","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127071278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177142024-02-15T03:05:31.245Z{"msg":"Upstairs opts: Upstairs UUID: 26117b2f-933c-4ebf-bd08-528a56f2add9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127115172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177152024-02-15T03:05:31.245Z{"msg":"Crucible stats registered with UUID: 26117b2f-933c-4ebf-bd08-528a56f2add9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127155137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177162024-02-15T03:05:31.245Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127179374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784}
177172024-02-15T03:05:31.245Z{"msg":"26117b2f-933c-4ebf-bd08-528a56f2add9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.12721386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177182024-02-15T03:05:31.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127235607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177192024-02-15T03:05:31.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127251435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177202024-02-15T03:05:31.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127281492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177212024-02-15T03:05:31.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127308048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177222024-02-15T03:05:31.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127333985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177232024-02-15T03:05:31.245Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.12737345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177242024-02-15T03:05:31.245Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127399147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177252024-02-15T03:05:31.245Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127437772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177262024-02-15T03:05:31.246Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127465568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177272024-02-15T03:05:31.246Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127585063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177282024-02-15T03:05:31.246Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.127623119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177292024-02-15T03:05:31.246Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:30.127746593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177302024-02-15T03:05:31.246Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:30.127793847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177312024-02-15T03:05:31.246Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:30.127998482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177322024-02-15T03:05:31.246Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.128026048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177332024-02-15T03:05:31.246Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.128040526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177342024-02-15T03:05:31.246Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.128123236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177352024-02-15T03:05:31.246Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.128466973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177362024-02-15T03:05:31.246Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.128505018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177372024-02-15T03:05:31.246Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.128636202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177382024-02-15T03:05:31.246Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.128668208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177392024-02-15T03:05:31.246Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.129021474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177402024-02-15T03:05:31.246Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.129107583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177412024-02-15T03:05:31.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.129648895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177422024-02-15T03:05:31.247Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.12969003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177432024-02-15T03:05:31.247Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.129708217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177442024-02-15T03:05:31.247Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.129730745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177452024-02-15T03:05:31.247Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.12976429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177462024-02-15T03:05:31.247Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:30.129781038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177472024-02-15T03:05:32.246Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.128775617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177482024-02-15T03:05:32.246Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.128880704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177492024-02-15T03:05:32.263Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.130102751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177502024-02-15T03:05:32.263Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.130139446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177512024-02-15T03:05:32.263Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.130663511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177522024-02-15T03:05:32.263Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131247158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177532024-02-15T03:05:32.263Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131288182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177542024-02-15T03:05:32.263Z{"msg":"[0]R flush_numbers[0..12]: [259, 259, 259, 258, 258, 258, 259, 259, 254, 259, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.1313074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177552024-02-15T03:05:32.263Z{"msg":"[0]R generation[0..12]: [158, 158, 158, 157, 157, 157, 158, 158, 155, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131337176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177562024-02-15T03:05:32.263Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131356014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177572024-02-15T03:05:32.264Z{"msg":"[1]R flush_numbers[0..12]: [259, 259, 259, 258, 258, 258, 259, 259, 254, 259, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131382541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177582024-02-15T03:05:32.264Z{"msg":"[1]R generation[0..12]: [158, 158, 158, 157, 157, 157, 158, 158, 155, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131400898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177592024-02-15T03:05:32.264Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131420556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177602024-02-15T03:05:32.264Z{"msg":"[2]R flush_numbers[0..12]: [259, 259, 259, 258, 258, 258, 259, 259, 254, 259, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131438124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177612024-02-15T03:05:32.264Z{"msg":"[2]R generation[0..12]: [158, 158, 158, 157, 157, 157, 158, 158, 155, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131470799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177622024-02-15T03:05:32.264Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]",Wait for a query_work_queue command to finish before sending IO
177632024-02-15T03:05:32.264Z"v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131492647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177642024-02-15T03:05:32.264Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131517664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177652024-02-15T03:05:32.264Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131535222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177662024-02-15T03:05:32.264Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131552539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177672024-02-15T03:05:32.265Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131569317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"":"downstairs","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177682024-02-15T03:05:32.265Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131586675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177692024-02-15T03:05:32.265Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131609962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"0","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177702024-02-15T03:05:32.265Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.13162707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"1","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177712024-02-15T03:05:32.265Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131651817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"client":"2","":"downstairs","session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177722024-02-15T03:05:32.265Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131668885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177732024-02-15T03:05:32.265Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131698911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177742024-02-15T03:05:32.265Z{"msg":"26117b2f-933c-4ebf-bd08-528a56f2add9 is now active with session: daa67e2b-6ff3-4e5a-bbc1-2f060ea66818","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131715479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177752024-02-15T03:05:32.265Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131730427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784,"session_id":"daa67e2b-6ff3-4e5a-bbc1-2f060ea66818"}
177762024-02-15T03:05:32.266Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:31.131745855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784}
177772024-02-15T03:05:32.266ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
177782024-02-15T03:05:32.266ZLoading write count information from file "/tmp/test_repair_verify.data"
177792024-02-15T03:05:32.266ZRun Repair workload
177802024-02-15T03:05:32.266Z01/30 Write block 193 len 20480 data: 18 18 19 15 13
177812024-02-15T03:05:32.266Z02/30 Write block 222 len 36864 data: 14 12 11 11 11 9 11 9 8
177822024-02-15T03:05:32.266Z03/30 Write block 203 len 12288 data: 7 8 10
177832024-02-15T03:05:32.266Z04/30 Read block 423 len 36864
177842024-02-15T03:05:32.266Z05/30 Write block 506 len 20480 data: 13 9 9 14 13
177852024-02-15T03:05:32.266Z06/30 Flush
177862024-02-15T03:05:32.266Z07/30 Write block 481 len 40960 data: 15 14 10 12 10 13 14 10 13 15
177872024-02-15T03:05:32.267Z08/30 Read block 132 len 16384
177882024-02-15T03:05:32.271Z09/30 Write block 174 len 16384 data: 6 7 7 9
177892024-02-15T03:05:32.272Z10/30 Read block 139 len 20480
177902024-02-15T03:05:32.272Z11/30 Write block 181 len 8192 data: 9 7
177912024-02-15T03:05:32.272Z12/30 Write block 211 len 4096 data: 11
177922024-02-15T03:05:32.272Z13/30 Write block 324 len 12288 data: 11 13 14
177932024-02-15T03:05:32.272Z14/30 Read block 447 len 12288
177942024-02-15T03:05:32.272Z15/30 Read block 289 len 28672
177952024-02-15T03:05:32.273Z16/30 Read block 400 len 40960
177962024-02-15T03:05:32.273Z17/30 Read block 578 len 24576
177972024-02-15T03:05:32.273Z18/30 Write block 529 len 36864 data: 14 14 10 13 11 11 13 10 13
177982024-02-15T03:05:32.273Z19/30 Write block 239 len 28672 data: 14 14 15 18 13 11 11
177992024-02-15T03:05:32.273Z20/30 Read block 64 len 24576
178002024-02-15T03:05:32.273Z21/30 Write block 257 len 32768 data: 14 14 18 15 12 8 8 8
178012024-02-15T03:05:32.273Z22/30 Read block 363 len 12288
178022024-02-15T03:05:32.273Z23/30 Write block 254 len 36864 data: 13 11 12 15 15 19 16 13 9
178032024-02-15T03:05:32.273Z24/30 Write block 149 len 36864 data: 10 11 11 10 12 12 12 17 17
178042024-02-15T03:05:32.273Z25/30 Read block 180 len 40960
178052024-02-15T03:05:32.273Z26/30 Flush
178062024-02-15T03:05:32.278Z27/30 Write block 309 len 32768 data: 14 15 14 10 11 11 11 12
178072024-02-15T03:05:32.278Z28/30 Write block 566 len 32768 data: 11 11 10 9 11 12 12 15
178082024-02-15T03:05:32.278Z29/30 Read block 210 len 24576
178092024-02-15T03:05:32.294Z30/30 Write block 329 len 36864 data: 13 14 16 15 15 18 17 15 15
178102024-02-15T03:05:32.294Z----------------------------------------------------------------
178112024-02-15T03:05:32.294Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
178122024-02-15T03:05:32.294ZStates: Active Active Active
178132024-02-15T03:05:32.294ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178142024-02-15T03:05:32.294Z 1 Acked 1000 Write 5 Done Sent Done false
178152024-02-15T03:05:32.294Z 2 Acked 1001 Write 9 Done Sent Done false
178162024-02-15T03:05:32.295Z 3 Acked 1002 Write 3 Done Sent Done false
178172024-02-15T03:05:32.295Z 4 Acked 1003 Read 9 Done Sent Done false
178182024-02-15T03:05:32.295Z 5 Acked 1004 Write 5 Done Sent Done false
178192024-02-15T03:05:32.295Z 6 Acked 1005 Flush 0 Done Sent Done false
178202024-02-15T03:05:32.295Z 7 Acked 1006 Write 10 Done Sent Done false
178212024-02-15T03:05:32.295Z 8 Acked 1007 Read 4 Done Sent Done false
178222024-02-15T03:05:32.295Z 9 Acked 1008 Write 4 Done Sent Done false
178232024-02-15T03:05:32.296Z 10 Acked 1009 Read 5 Done Sent Done false
178242024-02-15T03:05:32.296Z 11 Acked 1010 Write 2 Done Sent Done false
178252024-02-15T03:05:32.296Z 12 Acked 1011 Write 1 Done Sent Done false
178262024-02-15T03:05:32.296Z 13 Acked 1012 Write 3 Done Sent Done false
178272024-02-15T03:05:32.296Z 14 Acked 1013 Read 3 Done Sent Done false
178282024-02-15T03:05:32.296Z 15 Acked 1014 Read 7 Done Sent Done false
178292024-02-15T03:05:32.296Z 16 Acked 1015 Read 10 Done Sent Done false
178302024-02-15T03:05:32.296Z 17 Acked 1016 Read 6 Done Sent Done false
178312024-02-15T03:05:32.297Z 18 Acked 1017 Write 9 Done Sent Done false
178322024-02-15T03:05:32.297Z 19 Acked 1018 Write 7 Done Sent Done false
178332024-02-15T03:05:32.297Z 20 Acked 1019 Read 6 Done Sent Done false
178342024-02-15T03:05:32.297Z 21 Acked 1020 Write 8 Done Sent Done false
178352024-02-15T03:05:32.297Z 22 Acked 1021 Read 3 Done Sent Done false
178362024-02-15T03:05:32.297Z 23 Acked 1022 Write 9 Done Sent Done false
178372024-02-15T03:05:32.297Z 24 Acked 1023 Write 9 Done Sent Done false
178382024-02-15T03:05:32.297Z 25 Acked 1024 Read 10 Done Sent Done false
178392024-02-15T03:05:32.298Z 26 Acked 1025 Flush 0 Done Sent Done false
178402024-02-15T03:05:32.298Z 27 Acked 1026 Write 8 Done Sent Done false
178412024-02-15T03:05:32.298Z 28 Acked 1027 Write 8 Done Sent Done false
178422024-02-15T03:05:32.298Z 29 Acked 1028 Read 6 Done Sent Done false
178432024-02-15T03:05:32.298Z 30 Acked 1029 Write 9 Sent Sent Sent false
178442024-02-15T03:05:32.298Z STATES DS:0 DS:1 DS:2 TOTAL
178452024-02-15T03:05:32.299Z New 0 0 0 0
178462024-02-15T03:05:32.299Z Sent 1 30 1 32
178472024-02-15T03:05:32.299Z Done 29 0 29 58
178482024-02-15T03:05:32.299Z Skipped 0 0 0 0
178492024-02-15T03:05:32.299Z Error 0 0 0 0
178502024-02-15T03:05:32.299ZLast Flush: 1025 0 1025
178512024-02-15T03:05:32.299ZDownstairs last five completed:
178522024-02-15T03:05:32.300ZUpstairs last five completed: 30 29 28 27 26
178532024-02-15T03:05:32.300Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:31.16738605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1784}
178542024-02-15T03:05:32.300ZWrote out file "/tmp/test_repair_verify.data"
178552024-02-15T03:05:32.300Z
178562024-02-15T03:05:32.300ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
178572024-02-15T03:05:32.313ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178582024-02-15T03:05:32.313Z 0 000-019 158 158 158 259 259 259 F F F
178592024-02-15T03:05:32.314Z 1 020-039 158 158 158 259 259 259 F F F
178602024-02-15T03:05:32.314Z 2 040-059 158 158 158 259 259 259 F F F
178612024-02-15T03:05:32.314Z 3 060-079 157 157 157 258 258 258 F F F
178622024-02-15T03:05:32.314Z 4 080-099 157 157 157 258 258 258 F F F
178632024-02-15T03:05:32.314Z 5 100-119 157 157 157 258 258 258 F F F
178642024-02-15T03:05:32.314Z 6 120-139 158 158 158 259 259 259 F F F
178652024-02-15T03:05:32.315Z 7 140-159 160 158 160 262 259 262 F F F <---
178662024-02-15T03:05:32.315Z 8 160-179 160 155 160 262 254 262 F F F <---
178672024-02-15T03:05:32.315Z 9 180-199 160 158 160 262 259 262 F F F <---
178682024-02-15T03:05:32.315Z 10 200-219 160 158 160 262 259 262 F F F <---
178692024-02-15T03:05:32.315Z 11 220-239 160 158 160 262 259 262 F F F <---
178702024-02-15T03:05:32.316Z 12 240-259 160 155 160 262 254 262 F F F <---
178712024-02-15T03:05:32.316Z 13 260-279 160 159 160 262 260 262 F F F <---
178722024-02-15T03:05:32.316Z 14 280-299 158 158 158 259 259 259 F F F
178732024-02-15T03:05:32.316Z 15 300-319 158 158 158 259 259 259 T F T <---
178742024-02-15T03:05:32.316Z 16 320-339 160 157 160 262 258 262 T F T <---
178752024-02-15T03:05:32.316Z 17 340-359 159 159 159 260 260 260 F F F
178762024-02-15T03:05:32.316Z 18 360-379 153 153 153 248 248 248 F F F
178772024-02-15T03:05:32.316Z 19 380-399 158 158 158 259 259 259 F F F
178782024-02-15T03:05:32.317Z 20 400-419 158 158 158 259 259 259 F F F
178792024-02-15T03:05:32.317Z 21 420-439 148 148 148 241 241 241 F F F
178802024-02-15T03:05:32.317Z 22 440-459 157 157 157 258 258 258 F F F
178812024-02-15T03:05:32.317Z 23 460-479 156 156 156 255 255 255 F F F
178822024-02-15T03:05:32.317Z 24 480-499 160 157 160 262 258 262 F F F <---
178832024-02-15T03:05:32.317Z 25 500-519 160 159 160 261 260 261 F F F <---
178842024-02-15T03:05:32.318Z 26 520-539 160 158 160 262 259 262 F F F <---
178852024-02-15T03:05:32.318Z 27 540-559 159 159 159 260 260 260 F F F
178862024-02-15T03:05:32.318Z 28 560-579 158 158 158 259 259 259 T F T <---
178872024-02-15T03:05:32.318Z 29 580-599 154 154 154 251 251 251 F F F
178882024-02-15T03:05:32.318ZMax gen: 160, Max flush: 262
178892024-02-15T03:05:32.318ZError: Difference in extent metadata found!
178902024-02-15T03:05:32.318ZA Difference in extent metadata is expected here
178912024-02-15T03:05:32.319ZOn loop 80
178922024-02-15T03:05:32.319Z
178932024-02-15T03:05:32.319ZVerifying data now
178942024-02-15T03:05:36.967ZNo change after verify
178952024-02-15T03:05:36.967ZLoop: 80 Downstairs dump after verify (and repair):
178962024-02-15T03:05:36.983ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178972024-02-15T03:05:36.984Z 0 000-019 158 158 158 259 259 259 F F F
178982024-02-15T03:05:36.984Z 1 020-039 158 158 158 259 259 259 F F F
178992024-02-15T03:05:36.984Z 2 040-059 158 158 158 259 259 259 F F F
179002024-02-15T03:05:36.984Z 3 060-079 157 157 157 258 258 258 F F F
179012024-02-15T03:05:36.984Z 4 080-099 157 157 157 258 258 258 F F F
179022024-02-15T03:05:36.984Z 5 100-119 157 157 157 258 258 258 F F F
179032024-02-15T03:05:36.985Z 6 120-139 158 158 158 259 259 259 F F F
179042024-02-15T03:05:36.985Z 7 140-159 160 160 160 262 262 262 F F F
179052024-02-15T03:05:36.985Z 8 160-179 160 160 160 262 262 262 F F F
179062024-02-15T03:05:36.985Z 9 180-199 160 160 160 262 262 262 F F F
179072024-02-15T03:05:37.059Z 10 200-219 160 160 160 262 262 262 F F F
179082024-02-15T03:05:37.059Z 11 220-239 160 160 160 262 262 262 F F F
179092024-02-15T03:05:37.060Z 12 240-259 160 160 160 262 262 262 F F F
179102024-02-15T03:05:37.060Z 13 260-279 160 160 160 262 262 262 F F F
179112024-02-15T03:05:37.060Z 14 280-299 158 158 158 259 259 259 F F F
179122024-02-15T03:05:37.060Z 15 300-319 161 161 161 263 263 263 F F F
179132024-02-15T03:05:37.060Z 16 320-339 161 161 161 263 263 263 F F F
179142024-02-15T03:05:37.060Z 17 340-359 159 159 159 260 260 260 F F F
179152024-02-15T03:05:37.060Z 18 360-379 153 153 153 248 248 248 F F F
179162024-02-15T03:05:37.060Z 19 380-399 158 158 158 259 259 259 F F F
179172024-02-15T03:05:37.060Z 20 400-419 158 158 158 259 259 259 F F F
179182024-02-15T03:05:37.060Z 21 420-439 148 148 148 241 241 241 F F F
179192024-02-15T03:05:37.060Z 22 440-459 157 157 157 258 258 258 F F F
179202024-02-15T03:05:37.060Z 23 460-479 156 156 156 255 255 255 F F F
179212024-02-15T03:05:37.061Z 24 480-499 160 160 160 262 262 262 F F F
179222024-02-15T03:05:37.061Z 25 500-519 160 160 160 261 261 261 F F F
179232024-02-15T03:05:37.061Z 26 520-539 160 160 160 262 262 262 F F F
179242024-02-15T03:05:37.061Z 27 540-559 159 159 159 260 260 260 F F F
179252024-02-15T03:05:37.061Z 28 560-579 161 161 161 263 263 263 F F F
179262024-02-15T03:05:37.061Z 29 580-599 154 154 154 251 251 251 F F F
179272024-02-15T03:05:37.061ZMax gen: 161, Max flush: 263
179282024-02-15T03:05:37.061Z
179292024-02-15T03:05:37.062ZBegin loop 81
179302024-02-15T03:05:37.062ZDownstairs to restart: 2
179312024-02-15T03:05:37.071Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.893487493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792}
179322024-02-15T03:05:37.071Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.893735012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792}
179332024-02-15T03:05:37.071Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.893755739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792}
179342024-02-15T03:05:37.072Z{"msg":"Crucible 8a044509-6d8e-43ee-a1e4-285ae0f59b88 has session id: 3279f1be-050f-446e-a4c4-76c360ad0cba","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.893769698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179352024-02-15T03:05:37.072ZCrucible runtime is spawned
179362024-02-15T03:05:37.072Z{"msg":"Upstairs opts: Upstairs UUID: 8a044509-6d8e-43ee-a1e4-285ae0f59b88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.893818422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179372024-02-15T03:05:37.072Z{"msg":"Crucible stats registered with UUID: 8a044509-6d8e-43ee-a1e4-285ae0f59b88","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.893866436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179382024-02-15T03:05:37.072Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.893890862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792}
179392024-02-15T03:05:37.072Z{"msg":"8a044509-6d8e-43ee-a1e4-285ae0f59b88 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894052502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179402024-02-15T03:05:37.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894079219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179412024-02-15T03:05:37.072Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894094777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179422024-02-15T03:05:37.073Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894108445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179432024-02-15T03:05:37.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894134562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179442024-02-15T03:05:37.073Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.89415182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179452024-02-15T03:05:37.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894190995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179462024-02-15T03:05:37.073Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894208403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179472024-02-15T03:05:37.073Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894244848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179482024-02-15T03:05:37.073Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894280544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179492024-02-15T03:05:37.073Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:35.894433305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179502024-02-15T03:05:37.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894480899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179512024-02-15T03:05:37.073Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:35.894513144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179522024-02-15T03:05:37.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894553479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179532024-02-15T03:05:37.073Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:35.894575217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179542024-02-15T03:05:37.073Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894595654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179552024-02-15T03:05:37.073Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894618621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179562024-02-15T03:05:37.074Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.894642608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179572024-02-15T03:05:37.074Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.895140996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179582024-02-15T03:05:37.074Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.895167962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179592024-02-15T03:05:37.074Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.89518284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179602024-02-15T03:05:37.074Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.895213817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179612024-02-15T03:05:37.074Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.895804593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179622024-02-15T03:05:37.074Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.895859606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179632024-02-15T03:05:37.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.896376801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179642024-02-15T03:05:37.074Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.896412606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179652024-02-15T03:05:37.074Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.896427924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179662024-02-15T03:05:37.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.896447992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179672024-02-15T03:05:37.074Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.89646217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179682024-02-15T03:05:37.074Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:35.896482488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179692024-02-15T03:05:38.035Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.895156777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179702024-02-15T03:05:38.035Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.895260234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179712024-02-15T03:05:38.043Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.896161981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179722024-02-15T03:05:38.043Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.896201996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179732024-02-15T03:05:38.043Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.89673092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179742024-02-15T03:05:38.043Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897324475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179752024-02-15T03:05:38.043Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897384528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179762024-02-15T03:05:38.043Z{"msg":"[0]R flush_numbers[0..12]: [259, 259, 259, 258, 258, 258, 259, 262, 262, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897422933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179772024-02-15T03:05:38.044Z{"msg":"[0]R generation[0..12]: [158, 158, 158, 157, 157, 157, 158, 160, 160, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897450869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179782024-02-15T03:05:38.044Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897485115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179792024-02-15T03:05:38.044Z{"msg":"[1]R flush_numbers[0..12]: [259, 259, 259, 258, 258, 258, 259, 262, 262, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897512042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179802024-02-15T03:05:38.044Z{"msg":"[1]R generation[0..12]: [158, 158, 158, 157, 157, 157, 158, 160, 160, 160, 160, 160]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
179812024-02-15T03:05:38.044Z,"time":"2024-02-15T03:05:36.897544727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179822024-02-15T03:05:38.044Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897571364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179832024-02-15T03:05:38.044Z{"msg":"[2]R flush_numbers[0..12]: [259, 259, 259, 258, 258, 258, 259, 262, 262, 262, 262, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897599921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179842024-02-15T03:05:38.044Z{"msg":"[2]R generation[0..12]: [158, 158, 158, 157, 157, 157, 158, 160, 160, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897624828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179852024-02-15T03:05:38.044Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897653224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179862024-02-15T03:05:38.044Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897677041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179872024-02-15T03:05:38.045Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897704518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179882024-02-15T03:05:38.045Z{"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897727275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179892024-02-15T03:05:38.045Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897754151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"":"downstairs","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179902024-02-15T03:05:38.045Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897777618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179912024-02-15T03:05:38.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897803795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"0","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179922024-02-15T03:05:38.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897827312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"1","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179932024-02-15T03:05:38.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897854059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"client":"2","":"downstairs","session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179942024-02-15T03:05:38.045Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897877696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179952024-02-15T03:05:38.045Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897904473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179962024-02-15T03:05:38.045Z{"msg":"8a044509-6d8e-43ee-a1e4-285ae0f59b88 is now active with session: 3279f1be-050f-446e-a4c4-76c360ad0cba","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897928059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179972024-02-15T03:05:38.046Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"timeRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179982024-02-15T03:05:38.046Z":"2024-02-15T03:05:36.897953606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792,"session_id":"3279f1be-050f-446e-a4c4-76c360ad0cba"}
179992024-02-15T03:05:38.046Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:36.897981693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792}
180002024-02-15T03:05:38.046ZLoading write count information from file "/tmp/test_repair_verify.data"
180012024-02-15T03:05:38.046ZRun Repair workload
180022024-02-15T03:05:38.046Z01/30 Write block 65 len 4096 data: 11
180032024-02-15T03:05:38.046Z02/30 Write block 523 len 12288 data: 7 5 5
180042024-02-15T03:05:38.047Z03/30 Write block 269 len 40960 data: 10 13 13 15 14 11 12 13 14 10
180052024-02-15T03:05:38.047Z04/30 Read block 23 len 28672
180062024-02-15T03:05:38.047Z05/30 Read block 515 len 28672
180072024-02-15T03:05:38.047Z06/30 Read block 77 len 12288
180082024-02-15T03:05:38.047Z07/30 Read block 409 len 28672
180092024-02-15T03:05:38.047Z08/30 Write block 203 len 12288 data: 8 9 11
180102024-02-15T03:05:38.047Z09/30 Read block 18 len 12288
180112024-02-15T03:05:38.047Z10/30 Write block 179 len 36864 data: 11 11 10 8 11 13 15 15 14
180122024-02-15T03:05:38.048Z11/30 Write block 384 len 24576 data: 15 14 13 14 15 16
180132024-02-15T03:05:38.048Z12/30 Write block 243 len 28672 data: 14 12 12 11 12 10 11
180142024-02-15T03:05:38.048Z13/30 Read block 484 len 28672
180152024-02-15T03:05:38.048Z14/30 Flush
180162024-02-15T03:05:38.048Z15/30 Write block 574 len 4096 data: 16
180172024-02-15T03:05:38.048Z16/30 Read block 454 len 40960
180182024-02-15T03:05:38.048Z17/30 Write block 65 len 32768 data: 12 11 13 12 12 12 13 13
180192024-02-15T03:05:38.048Z18/30 Write block 518 len 4096 data: 12
180202024-02-15T03:05:38.048Z19/30 Write block 581 len 12288 data: 10 12 11
180212024-02-15T03:05:38.048Z20/30 Write block 564 len 8192 data: 12 15
180222024-02-15T03:05:38.048Z21/30 Write block 165 len 4096 data: 13
180232024-02-15T03:05:38.048Z22/30 Write block 193 len 32768 data: 19 19 20 16 14 10 12 13
180242024-02-15T03:05:38.048Z23/30 Read block 565 len 16384
180252024-02-15T03:05:38.049Z24/30 Read block 558 len 8192
180262024-02-15T03:05:38.049Z25/30 Write block 410 len 12288 data: 20 20 18
180272024-02-15T03:05:38.049Z26/30 Read block 155 len 24576
180282024-02-15T03:05:38.049Z27/30 Read block 1 len 40960
180292024-02-15T03:05:38.049Z28/30 Read block 251 len 24576
180302024-02-15T03:05:38.049Z29/30 Write block 251 len 16384 data: 12 14 13 14
180312024-02-15T03:05:38.049Z30/30 Write block 515 len 4096 data: 12
180322024-02-15T03:05:38.049Z----------------------------------------------------------------
180332024-02-15T03:05:38.049Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
180342024-02-15T03:05:38.049ZStates: Active Active Active
180352024-02-15T03:05:38.049ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180362024-02-15T03:05:38.049Z 1 Acked 1000 Write 1 Done Done Sent false
180372024-02-15T03:05:38.050Z 2 Acked 1001 Write 3 Done Done Sent false
180382024-02-15T03:05:38.050Z 3 Acked 1002 Write 10 Done Done Sent false
180392024-02-15T03:05:38.050Z 4 Acked 1003 Read 7 Done Done Sent false
180402024-02-15T03:05:38.050Z 5 Acked 1004 Read 7 Done Done Sent false
180412024-02-15T03:05:38.050Z 6 Acked 1005 Read 3 Done Done Sent false
180422024-02-15T03:05:38.050Z 7 Acked 1006 Read 7 Done Done Sent false
180432024-02-15T03:05:38.050Z 8 Acked 1007 Write 3 Done Done Sent false
180442024-02-15T03:05:38.050Z 9 Acked 1008 Read 3 Done Done Sent false
180452024-02-15T03:05:38.050Z 10 Acked 1009 Write 9 Done Done Sent false
180462024-02-15T03:05:38.050Z 11 Acked 1010 Write 6 Done Done Sent false
180472024-02-15T03:05:38.050Z 12 Acked 1011 Write 7 Done Done Sent false
180482024-02-15T03:05:38.050Z 13 Acked 1012 Read 7 Done Done Sent false
180492024-02-15T03:05:38.050Z 14 Acked 1013 Flush 0 Done Done Sent false
180502024-02-15T03:05:38.050Z 15 Acked 1014 Write 1 Done Done Sent false
180512024-02-15T03:05:38.050Z 16 Acked 1015 Read 10 Done Done Sent false
180522024-02-15T03:05:38.050Z 17 Acked 1016 Write 8 Done Done Sent false
180532024-02-15T03:05:38.050Z 18 Acked 1017 Write 1 Done Done Sent false
180542024-02-15T03:05:38.051Z 19 Acked 1018 Write 3 Done Done Sent false
180552024-02-15T03:05:38.051Z 20 Acked 1019 Write 2 Done Done Sent false
180562024-02-15T03:05:38.051Z 21 Acked 1020 Write 1 Done Done Sent false
180572024-02-15T03:05:38.051Z 22 Acked 1021 Write 8 Done Done Sent false
180582024-02-15T03:05:38.051Z 23 Acked 1022 Read 4 Done Done Sent false
180592024-02-15T03:05:38.051Z 24 Acked 1023 Read 2 Done Done Sent false
180602024-02-15T03:05:38.051Z 25 Acked 1024 Write 3 Done Done Sent false
180612024-02-15T03:05:38.051Z 26 Acked 1025 Read 6 Done Done Sent false
180622024-02-15T03:05:38.051Z 27 Acked 1026 Read 10 Done Done Sent false
180632024-02-15T03:05:38.051Z 28 Acked 1027 Read 6 Done Done Sent false
180642024-02-15T03:05:38.051Z 29 Acked 1028 Write 4 Sent Sent Sent false
180652024-02-15T03:05:38.051Z 30 Acked 1029 Write 1 Sent Sent Sent false
180662024-02-15T03:05:38.051Z STATES DS:0 DS:1 DS:2 TOTAL
180672024-02-15T03:05:38.051Z New 0 0 0 0
180682024-02-15T03:05:38.051Z Sent 2 2 30 34
180692024-02-15T03:05:38.051Z Done 28 28 0 56
180702024-02-15T03:05:38.051Z Skipped 0 0 0 0
180712024-02-15T03:05:38.057Z Error 0 0 0 0
180722024-02-15T03:05:38.058ZLast Flush: 1013 1013 0
180732024-02-15T03:05:38.058ZDownstairs last five completed:
180742024-02-15T03:05:38.058ZUpstairs last five completed: 30 29 28 27 26
180752024-02-15T03:05:38.058Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:36.925690701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1792}
180762024-02-15T03:05:38.058ZWrote out file "/tmp/test_repair_verify.data"
180772024-02-15T03:05:38.058Z
180782024-02-15T03:05:38.058ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
180792024-02-15T03:05:38.069ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180802024-02-15T03:05:38.069Z 0 000-019 158 158 158 259 259 259 F F F
180812024-02-15T03:05:38.069Z 1 020-039 158 158 158 259 259 259 F F F
180822024-02-15T03:05:38.069Z 2 040-059 158 158 158 259 259 259 F F F
180832024-02-15T03:05:38.069Z 3 060-079 162 162 157 264 264 258 T T F <---
180842024-02-15T03:05:38.069Z 4 080-099 157 157 157 258 258 258 F F F
180852024-02-15T03:05:38.070Z 5 100-119 157 157 157 258 258 258 F F F
180862024-02-15T03:05:38.070Z 6 120-139 158 158 158 259 259 259 F F F
180872024-02-15T03:05:38.070Z 7 140-159 160 160 160 262 262 262 F F F
180882024-02-15T03:05:38.070Z 8 160-179 162 162 160 264 264 262 T T F <---
180892024-02-15T03:05:38.070Z 9 180-199 162 162 160 264 264 262 T T F <---
180902024-02-15T03:05:38.081Z 10 200-219 162 162 160 264 264 262 T T F <---
180912024-02-15T03:05:38.081Z 11 220-239 160 160 160 262 262 262 F F F
180922024-02-15T03:05:38.081Z 12 240-259 162 162 160 264 264 262 T T F <---
180932024-02-15T03:05:38.081Z 13 260-279 162 162 160 264 264 262 F F F <---
180942024-02-15T03:05:38.081Z 14 280-299 158 158 158 259 259 259 F F F
180952024-02-15T03:05:38.081Z 15 300-319 161 161 161 263 263 263 F F F
180962024-02-15T03:05:38.082Z 16 320-339 161 161 161 263 263 263 F F F
180972024-02-15T03:05:38.082Z 17 340-359 159 159 159 260 260 260 F F F
180982024-02-15T03:05:38.082Z 18 360-379 153 153 153 248 248 248 F F F
180992024-02-15T03:05:38.082Z 19 380-399 162 162 158 264 264 259 F F F <---
181002024-02-15T03:05:38.083Z 20 400-419 158 158 158 259 259 259 T T F <---
181012024-02-15T03:05:38.083Z 21 420-439 148 148 148 241 241 241 F F F
181022024-02-15T03:05:38.083Z 22 440-459 157 157 157 258 258 258 F F F
181032024-02-15T03:05:38.083Z 23 460-479 156 156 156 255 255 255 F F F
181042024-02-15T03:05:38.083Z 24 480-499 160 160 160 262 262 262 F F F
181052024-02-15T03:05:38.083Z 25 500-519 160 160 160 261 261 261 T T F <---
181062024-02-15T03:05:38.083Z 26 520-539 162 162 160 264 264 262 F F F <---
181072024-02-15T03:05:38.084Z 27 540-559 159 159 159 260 260 260 F F F
181082024-02-15T03:05:38.084Z 28 560-579 161 161 161 263 263 263 T T F <---
181092024-02-15T03:05:38.084Z 29 580-599 154 154 154 251 251 251 T T F <---
181102024-02-15T03:05:38.084ZMax gen: 162, Max flush: 264
181112024-02-15T03:05:38.084ZError: Difference in extent metadata found!
181122024-02-15T03:05:38.084ZA Difference in extent metadata is expected here
181132024-02-15T03:05:38.084ZOn loop 81
181142024-02-15T03:05:38.084Z
181152024-02-15T03:05:38.084ZVerifying data now
181162024-02-15T03:05:42.696ZNo change after verify
181172024-02-15T03:05:42.696ZLoop: 81 Downstairs dump after verify (and repair):
181182024-02-15T03:05:42.723ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181192024-02-15T03:05:42.723Z 0 000-019 158 158 158 259 259 259 F F F
181202024-02-15T03:05:42.723Z 1 020-039 158 158 158 259 259 259 F F F
181212024-02-15T03:05:42.723Z 2 040-059 158 158 158 259 259 259 F F F
181222024-02-15T03:05:42.723Z 3 060-079 163 163 163 265 265 265 F F F
181232024-02-15T03:05:42.723Z 4 080-099 157 157 157 258 258 258 F F F
181242024-02-15T03:05:42.724Z 5 100-119 157 157 157 258 258 258 F F F
181252024-02-15T03:05:42.724Z 6 120-139 158 158 158 259 259 259 F F F
181262024-02-15T03:05:42.724Z 7 140-159 160 160 160 262 262 262 F F F
181272024-02-15T03:05:42.724Z 8 160-179 163 163 163 265 265 265 F F F
181282024-02-15T03:05:42.724Z 9 180-199 163 163 163 265 265 265 F F F
181292024-02-15T03:05:42.724Z 10 200-219 163 163 163 265 265 265 F F F
181302024-02-15T03:05:42.724Z 11 220-239 160 160 160 262 262 262 F F F
181312024-02-15T03:05:42.724Z 12 240-259 163 163 163 265 265 265 F F F
181322024-02-15T03:05:42.724Z 13 260-279 162 162 162 264 264 264 F F F
181332024-02-15T03:05:42.724Z 14 280-299 158 158 158 259 259 259 F F F
181342024-02-15T03:05:42.724Z 15 300-319 161 161 161 263 263 263 F F F
181352024-02-15T03:05:42.724Z 16 320-339 161 161 161 263 263 263 F F F
181362024-02-15T03:05:42.724Z 17 340-359 159 159 159 260 260 260 F F F
181372024-02-15T03:05:42.725Z 18 360-379 153 153 153 248 248 248 F F F
181382024-02-15T03:05:42.725Z 19 380-399 162 162 162 264 264 264 F F F
181392024-02-15T03:05:42.725Z 20 400-419 163 163 163 265 265 265 F F F
181402024-02-15T03:05:42.725Z 21 420-439 148 148 148 241 241 241 F F F
181412024-02-15T03:05:42.725Z 22 440-459 157 157 157 258 258 258 F F F
181422024-02-15T03:05:42.725Z 23 460-479 156 156 156 255 255 255 F F F
181432024-02-15T03:05:42.725Z 24 480-499 160 160 160 262 262 262 F F F
181442024-02-15T03:05:42.725Z 25 500-519 163 163 163 265 265 265 F F F
181452024-02-15T03:05:42.725Z 26 520-539 162 162 162 264 264 264 F F F
181462024-02-15T03:05:42.726Z 27 540-559 159 159 159 260 260 260 F F F
181472024-02-15T03:05:42.726Z 28 560-579 163 163 163 265 265 265 F F F
181482024-02-15T03:05:42.726Z 29 580-599 163 163 163 265 265 265 F F F
181492024-02-15T03:05:42.726ZMax gen: 163, Max flush: 265
181502024-02-15T03:05:42.726Z
181512024-02-15T03:05:42.726ZBegin loop 82
181522024-02-15T03:05:42.726ZDownstairs to restart: 1
181532024-02-15T03:05:42.754Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619300736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800}
181542024-02-15T03:05:42.754Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619602098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800}
181552024-02-15T03:05:42.754Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619629434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800}
181562024-02-15T03:05:42.754ZCrucible runtime is spawned
181572024-02-15T03:05:42.755Z{"msg":"Crucible 6777edca-9938-440c-98df-2459f7beffb3 has session id: e380ed5b-d032-437f-9f7a-9e67bd0eb67d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619673039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181582024-02-15T03:05:42.755Z{"msg":"Upstairs opts: Upstairs UUID: 6777edca-9938-440c-98df-2459f7beffb3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619728352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181592024-02-15T03:05:42.755Z{"msg":"Crucible stats registered with UUID: 6777edca-9938-440c-98df-2459f7beffb3","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619754129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181602024-02-15T03:05:42.755Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619785475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800}
181612024-02-15T03:05:42.755Z{"msg":"6777edca-9938-440c-98df-2459f7beffb3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619896151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181622024-02-15T03:05:42.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619924567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181632024-02-15T03:05:42.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619940565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181642024-02-15T03:05:42.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.619978231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181652024-02-15T03:05:42.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620025865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181662024-02-15T03:05:42.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.62006371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181672024-02-15T03:05:42.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620097986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181682024-02-15T03:05:42.755Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620149579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181692024-02-15T03:05:42.756Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620180165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181702024-02-15T03:05:42.756Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620215451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181712024-02-15T03:05:42.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620243177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181722024-02-15T03:05:42.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620285462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181732024-02-15T03:05:42.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.62030392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181742024-02-15T03:05:42.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.620978685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181752024-02-15T03:05:42.756Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.621030239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181762024-02-15T03:05:42.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.621047806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181772024-02-15T03:05:42.756Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.621077233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181782024-02-15T03:05:42.756Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.62165555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181792024-02-15T03:05:42.757Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.621835488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181802024-02-15T03:05:42.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622170836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181812024-02-15T03:05:42.757Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622210511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181822024-02-15T03:05:42.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622226979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181832024-02-15T03:05:42.757Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622240987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181842024-02-15T03:05:42.757Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622255985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181852024-02-15T03:05:42.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622395688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181862024-02-15T03:05:42.757Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622432513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181872024-02-15T03:05:42.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622447831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181882024-02-15T03:05:42.757Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.622844252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181892024-02-15T03:05:42.758Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623317402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181902024-02-15T03:05:42.758Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623350798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181912024-02-15T03:05:42.758Z{"msg":"[0]R flush_numbers[0..12]: [259, 259, 259, 265, 258, 258, 259, 262, 265, 265, 265, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623367366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181922024-02-15T03:05:42.758Z{"msg":"[0]R generation[0..12]: [158, 158, 158, 163, 157, 157, 158, 160, 163, 163, 163, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623390463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181932024-02-15T03:05:42.758Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623405651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181942024-02-15T03:05:42.758Z{"msg":"[1]R flush_numbers[0..12]: [259, 259, 259, 265, 258, 258, 259, 262, 265, 265, 265, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623420529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181952024-02-15T03:05:42.758Z{"msg":"[1]R generation[0..12]: [158, 158, 158, 163, 157, 157, 158, 160, 163, 163, 163, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623434707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181962024-02-15T03:05:42.758Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623456615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181972024-02-15T03:05:42.758Z{"msg":"[2]R flush_numbers[0..12]: [259, 259, 259, 265, 258, 258, 259, 262, 265, 265, 265, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623471293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181982024-02-15T03:05:42.758Z{"msg":"[2]R generation[0..12]: [158, 158, 158, 163, 157, 157, 158, 160, 163, 163, 163, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623485871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
181992024-02-15T03:05:42.758Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623499849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182002024-02-15T03:05:42.758Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623520957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182012024-02-15T03:05:42.759Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
182022024-02-15T03:05:42.759Z2024-02-15T03:05:41.623535225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182032024-02-15T03:05:42.759Z{"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623552853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182042024-02-15T03:05:42.759Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623592168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"":"downstairs","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182052024-02-15T03:05:42.759Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623606226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182062024-02-15T03:05:42.759Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623619424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"0","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182072024-02-15T03:05:42.759Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623640052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"1","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182082024-02-15T03:05:42.759Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.62365443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"client":"2","":"downstairs","session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182092024-02-15T03:05:42.759Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623675017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182102024-02-15T03:05:42.759Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623688056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182112024-02-15T03:05:42.759Z{"msg":"6777edca-9938-440c-98df-2459f7beffb3 is now active with session: e380ed5b-d032-437f-9f7a-9e67bd0eb67d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623707343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182122024-02-15T03:05:42.760Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623720712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800,"session_id":"e380ed5b-d032-437f-9f7a-9e67bd0eb67d"}
182132024-02-15T03:05:42.760Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:41.623740039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800}
182142024-02-15T03:05:42.760ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182152024-02-15T03:05:42.760ZLoading write count information from file "/tmp/test_repair_verify.data"
182162024-02-15T03:05:42.760ZRun Repair workload
182172024-02-15T03:05:42.760Z01/30 Write block 373 len 36864 data: 12 10 10 11 13 14 12 11 12
182182024-02-15T03:05:42.760Z02/30 Write block 68 len 8192 data: 13 13
182192024-02-15T03:05:42.760Z03/30 Flush
182202024-02-15T03:05:42.761Z04/30 Write block 43 len 16384 data: 16 18 20 16
182212024-02-15T03:05:42.761Z05/30 Write block 124 len 20480 data: 9 11 13 13 13
182222024-02-15T03:05:42.761Z06/30 Flush
182232024-02-15T03:05:42.761Z07/30 Write block 202 len 12288 data: 8 9 10
182242024-02-15T03:05:42.761Z08/30 Read block 301 len 28672
182252024-02-15T03:05:42.791Z09/30 Read block 516 len 8192
182262024-02-15T03:05:42.792Z10/30 Read block 380 len 20480
182272024-02-15T03:05:42.792Z11/30 Flush
182282024-02-15T03:05:42.792Z12/30 Write block 580 len 4096 data: 14
182292024-02-15T03:05:42.792Z13/30 Read block 418 len 36864
182302024-02-15T03:05:42.792Z14/30 Write block 519 len 24576 data: 11 9 7 6 8 6
182312024-02-15T03:05:42.792Z15/30 Read block 114 len 40960
182322024-02-15T03:05:42.792Z16/30 Write block 247 len 28672 data: 13 11 12 12 13 15 14
182332024-02-15T03:05:42.792Z17/30 Flush
182342024-02-15T03:05:42.792Z18/30 Write block 578 len 40960 data: 15 13 15 11 13 12 11 12 12 10
182352024-02-15T03:05:42.793Z19/30 Read block 183 len 28672
182362024-02-15T03:05:42.793Z20/30 Flush
182372024-02-15T03:05:42.793Z21/30 Write block 459 len 40960 data: 10 13 12 12 9 9 11 14 14 13
182382024-02-15T03:05:42.793Z22/30 Read block 400 len 20480
182392024-02-15T03:05:42.793Z23/30 Read block 27 len 8192
182402024-02-15T03:05:42.793Z24/30 Read block 245 len 40960
182412024-02-15T03:05:42.793Z25/30 Write block 177 len 32768 data: 10 10 12 12 11 9 12 14
182422024-02-15T03:05:42.793Z26/30 Write block 500 len 4096 data: 14
182432024-02-15T03:05:42.793Z27/30 Read block 99 len 4096
182442024-02-15T03:05:42.793Z28/30 Write block 292 len 20480 data: 19 17 17 15 14
182452024-02-15T03:05:42.793Z29/30 Write block 98 len 16384 data: 17 15 12 13
182462024-02-15T03:05:42.794Z30/30 Write block 68 len 4096 data: 14
182472024-02-15T03:05:42.794Z----------------------------------------------------------------
182482024-02-15T03:05:42.794Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:24
182492024-02-15T03:05:42.794ZStates: Active Active Active
182502024-02-15T03:05:42.794ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182512024-02-15T03:05:42.794Z 7 Acked 1006 Write 3 Done Done Done false
182522024-02-15T03:05:42.794Z 8 Acked 1007 Read 7 Done Done Done false
182532024-02-15T03:05:42.794Z 9 Acked 1008 Read 2 Done Sent Done false
182542024-02-15T03:05:42.794Z 10 Acked 1009 Read 5 Done Sent Done false
182552024-02-15T03:05:42.794Z 11 Acked 1010 Flush 0 Done Sent Done false
182562024-02-15T03:05:42.795Z 12 Acked 1011 Write 1 Done Sent Done false
182572024-02-15T03:05:42.795Z 13 Acked 1012 Read 9 Done Sent Done false
182582024-02-15T03:05:42.795Z 14 Acked 1013 Write 6 Done Sent Done false
182592024-02-15T03:05:42.795Z 15 Acked 1014 Read 10 Done Sent Done false
182602024-02-15T03:05:42.795Z 16 Acked 1015 Write 7 Done Sent Done false
182612024-02-15T03:05:42.795Z 17 Acked 1016 Flush 0 Done Sent Done false
182622024-02-15T03:05:42.795Z 18 Acked 1017 Write 10 Done Sent Done false
182632024-02-15T03:05:42.795Z 19 Acked 1018 Read 7 Done Sent Done false
182642024-02-15T03:05:42.795Z 20 Acked 1019 Flush 0 Done Sent Done false
182652024-02-15T03:05:42.795Z 21 Acked 1020 Write 10 Done Sent Done false
182662024-02-15T03:05:42.795Z 22 Acked 1021 Read 5 Done Sent Done false
182672024-02-15T03:05:42.796Z 23 Acked 1022 Read 2 Done Sent Done false
182682024-02-15T03:05:42.796Z 24 Acked 1023 Read 10 Done Sent Done false
182692024-02-15T03:05:42.796Z 25 Acked 1024 Write 8 Done Sent Done false
182702024-02-15T03:05:42.796Z 26 Acked 1025 Write 1 Done Sent Done false
182712024-02-15T03:05:42.796Z 27 Acked 1026 Read 1 Done Sent Done false
182722024-02-15T03:05:42.796Z 28 Acked 1027 Write 5 Done Sent Done false
182732024-02-15T03:05:42.796Z 29 Acked 1028 Write 4 Sent Sent Sent false
182742024-02-15T03:05:42.796Z 30 Acked 1029 Write 1 Sent Sent Sent false
182752024-02-15T03:05:42.796Z STATES DS:0 DS:1 DS:2 TOTAL
182762024-02-15T03:05:42.796Z New 0 0 0 0
182772024-02-15T03:05:42.796Z Sent 2 22 2 26
182782024-02-15T03:05:42.797Z Done 22 2 22 46
182792024-02-15T03:05:42.797Z Skipped 0 0 0 0
182802024-02-15T03:05:42.797Z Error 0 0 0 0
182812024-02-15T03:05:42.797ZLast Flush: 1019 1005 1019
182822024-02-15T03:05:42.797ZDownstairs last five completed: 1005 1004 1003 1002 1001
182832024-02-15T03:05:42.797ZUpstairs last five completed: 30 29 28 27 26
182842024-02-15T03:05:42.797Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:41.657876522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1800}
182852024-02-15T03:05:42.797ZWrote out file "/tmp/test_repair_verify.data"
182862024-02-15T03:05:42.797Z
182872024-02-15T03:05:42.819ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182882024-02-15T03:05:42.819ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182892024-02-15T03:05:42.819Z 0 000-019 158 158 158 259 259 259 F F F
182902024-02-15T03:05:42.819Z 1 020-039 158 158 158 259 259 259 F F F
182912024-02-15T03:05:42.819Z 2 040-059 164 164 164 267 267 267 F F F
182922024-02-15T03:05:42.820Z 3 060-079 164 164 164 266 266 266 T F T <---
182932024-02-15T03:05:42.820Z 4 080-099 157 157 157 258 258 258 T F T <---
182942024-02-15T03:05:42.820Z 5 100-119 157 157 157 258 258 258 T F T <---
182952024-02-15T03:05:42.820Z 6 120-139 164 164 164 267 267 267 F F F
182962024-02-15T03:05:42.820Z 7 140-159 160 160 160 262 262 262 F F F
182972024-02-15T03:05:42.820Z 8 160-179 163 163 163 265 265 265 T F T <---
182982024-02-15T03:05:42.820Z 9 180-199 163 163 163 265 265 265 T F T <---
182992024-02-15T03:05:42.820Z 10 200-219 164 163 164 268 265 268 F T F <---
183002024-02-15T03:05:42.820Z 11 220-239 160 160 160 262 262 262 F F F
183012024-02-15T03:05:42.820Z 12 240-259 164 163 164 269 265 269 F F F <---
183022024-02-15T03:05:42.820Z 13 260-279 162 162 162 264 264 264 F F F
183032024-02-15T03:05:42.820Z 14 280-299 158 158 158 259 259 259 T F T <---
183042024-02-15T03:05:42.821Z 15 300-319 161 161 161 263 263 263 F F F
183052024-02-15T03:05:42.821Z 16 320-339 161 161 161 263 263 263 F F F
183062024-02-15T03:05:42.821Z 17 340-359 159 159 159 260 260 260 F F F
183072024-02-15T03:05:42.821Z 18 360-379 164 164 164 266 266 266 F F F
183082024-02-15T03:05:42.821Z 19 380-399 164 164 164 266 266 266 F F F
183092024-02-15T03:05:42.821Z 20 400-419 163 163 163 265 265 265 F F F
183102024-02-15T03:05:42.821Z 21 420-439 148 148 148 241 241 241 F F F
183112024-02-15T03:05:42.822Z 22 440-459 157 157 157 258 258 258 T F T <---
183122024-02-15T03:05:42.822Z 23 460-479 156 156 156 255 255 255 T F T <---
183132024-02-15T03:05:42.822Z 24 480-499 160 160 160 262 262 262 F F F
183142024-02-15T03:05:42.822Z 25 500-519 164 163 164 269 265 269 T F T <---
183152024-02-15T03:05:42.822Z 26 520-539 164 162 164 269 264 269 F F F <---
183162024-02-15T03:05:42.822Z 27 540-559 159 159 159 260 260 260 F F F
183172024-02-15T03:05:42.822Z 28 560-579 164 163 164 270 265 270 F F F <---
183182024-02-15T03:05:42.822Z 29 580-599 164 163 164 270 265 270 F F F <---
183192024-02-15T03:05:42.822ZMax gen: 164, Max flush: 270
183202024-02-15T03:05:42.822ZError: Difference in extent metadata found!
183212024-02-15T03:05:42.822ZA Difference in extent metadata is expected here
183222024-02-15T03:05:42.822ZOn loop 82
183232024-02-15T03:05:42.822Z
183242024-02-15T03:05:42.822ZVerifying data now
183252024-02-15T03:05:47.536ZNo change after verify
183262024-02-15T03:05:47.536ZLoop: 82 Downstairs dump after verify (and repair):
183272024-02-15T03:05:47.544ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183282024-02-15T03:05:47.545Z 0 000-019 158 158 158 259 259 259 F F F
183292024-02-15T03:05:47.545Z 1 020-039 158 158 158 259 259 259 F F F
183302024-02-15T03:05:47.545Z 2 040-059 164 164 164 267 267 267 F F F
183312024-02-15T03:05:47.545Z 3 060-079 165 165 165 271 271 271 F F F
183322024-02-15T03:05:47.545Z 4 080-099 165 165 165 271 271 271 F F F
183332024-02-15T03:05:47.545Z 5 100-119 165 165 165 271 271 271 F F F
183342024-02-15T03:05:47.583Z 6 120-139 164 164 164 267 267 267 F F F
183352024-02-15T03:05:47.583Z 7 140-159 160 160 160 262 262 262 F F F
183362024-02-15T03:05:47.583Z 8 160-179 165 165 165 271 271 271 F F F
183372024-02-15T03:05:47.583Z 9 180-199 165 165 165 271 271 271 F F F
183382024-02-15T03:05:47.583Z 10 200-219 164 164 164 268 268 268 F F F
183392024-02-15T03:05:47.583Z 11 220-239 160 160 160 262 262 262 F F F
183402024-02-15T03:05:47.583Z 12 240-259 164 164 164 269 269 269 F F F
183412024-02-15T03:05:47.583Z 13 260-279 162 162 162 264 264 264 F F F
183422024-02-15T03:05:47.583Z 14 280-299 165 165 165 271 271 271 F F F
183432024-02-15T03:05:47.583Z 15 300-319 161 161 161 263 263 263 F F F
183442024-02-15T03:05:47.583Z 16 320-339 161 161 161 263 263 263 F F F
183452024-02-15T03:05:47.584Z 17 340-359 159 159 159 260 260 260 F F F
183462024-02-15T03:05:47.584Z 18 360-379 164 164 164 266 266 266 F F F
183472024-02-15T03:05:47.584Z 19 380-399 164 164 164 266 266 266 F F F
183482024-02-15T03:05:47.584Z 20 400-419 163 163 163 265 265 265 F F F
183492024-02-15T03:05:47.584Z 21 420-439 148 148 148 241 241 241 F F F
183502024-02-15T03:05:47.584Z 22 440-459 165 165 165 271 271 271 F F F
183512024-02-15T03:05:47.584Z 23 460-479 165 165 165 271 271 271 F F F
183522024-02-15T03:05:47.584Z 24 480-499 160 160 160 262 262 262 F F F
183532024-02-15T03:05:47.584Z 25 500-519 165 165 165 271 271 271 F F F
183542024-02-15T03:05:47.584Z 26 520-539 164 164 164 269 269 269 F F F
183552024-02-15T03:05:47.584Z 27 540-559 159 159 159 260 260 260 F F F
183562024-02-15T03:05:47.584Z 28 560-579 164 164 164 270 270 270 F F F
183572024-02-15T03:05:47.584Z 29 580-599 164 164 164 270 270 270 F F F
183582024-02-15T03:05:47.584ZMax gen: 165, Max flush: 271
183592024-02-15T03:05:47.585Z
183602024-02-15T03:05:47.585ZBegin loop 83
183612024-02-15T03:05:47.585ZDownstairs to restart: 0
183622024-02-15T03:05:47.606Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.453768442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808}
183632024-02-15T03:05:47.606Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454030069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808}
183642024-02-15T03:05:47.606Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454048347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808}
183652024-02-15T03:05:47.606Z{"msg":"Crucible c236053e-4460-433e-ae01-5e332f808347 has session id: 3e126cc2-641f-4497-8c71-ba7036bd31d9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454070884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183662024-02-15T03:05:47.606Z{"msg":"Upstairs opts: Upstairs UUID: c236053e-4460-433e-ae01-5e332f808347, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454094061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183672024-02-15T03:05:47.606Z{"msg":"Crucible stats registered with UUID: c236053e-4460-433e-ae01-5e332f808347","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454117098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183682024-02-15T03:05:47.606ZCrucible runtime is spawned
183692024-02-15T03:05:47.606Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454368446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808}
183702024-02-15T03:05:47.606Z{"msg":"c236053e-4460-433e-ae01-5e332f808347 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454399083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183712024-02-15T03:05:47.606Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454413271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183722024-02-15T03:05:47.606Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454441388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183732024-02-15T03:05:47.607Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454476103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183742024-02-15T03:05:47.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454507799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183752024-02-15T03:05:47.607Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454549094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183762024-02-15T03:05:47.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454573911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183772024-02-15T03:05:47.607Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454623894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183782024-02-15T03:05:47.607Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.45466063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183792024-02-15T03:05:47.607Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454690156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183802024-02-15T03:05:47.607Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:46.454807311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183812024-02-15T03:05:47.607Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:46.454858215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183822024-02-15T03:05:47.607Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454904019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183832024-02-15T03:05:47.607Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454933636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183842024-02-15T03:05:47.607Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:46.454950594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183852024-02-15T03:05:47.607Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.454964202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183862024-02-15T03:05:47.607Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.45497839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183872024-02-15T03:05:47.607Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.455133621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183882024-02-15T03:05:47.607Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.455578615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183892024-02-15T03:05:47.607Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.455609481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183902024-02-15T03:05:47.607Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.455624669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183912024-02-15T03:05:47.607Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.455689561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183922024-02-15T03:05:47.607Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456196757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183932024-02-15T03:05:47.607Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456346689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183942024-02-15T03:05:47.607Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456789613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183952024-02-15T03:05:47.607Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456826878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183962024-02-15T03:05:47.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456844326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183972024-02-15T03:05:47.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456866413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183982024-02-15T03:05:47.608Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456888051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
183992024-02-15T03:05:47.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:46.456902589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184002024-02-15T03:05:48.571Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.456816713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184012024-02-15T03:05:48.571Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.456941067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184022024-02-15T03:05:48.577Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.457843494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184032024-02-15T03:05:48.577Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.457884279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184042024-02-15T03:05:48.577Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.458374158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184052024-02-15T03:05:48.577Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.458978752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184062024-02-15T03:05:48.577Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459046044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184072024-02-15T03:05:48.577Z{"msg":"[0]R flush_numbers[0..12]: [259, 259, 267, 271, 271, 271, 267, 262, 271, 271, 268, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459081659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184082024-02-15T03:05:48.577Z{"msg":"[0]R generation[0..12]: [158, 158, 164, 165, 165, 165, 164, 160, 165, 165, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459121114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184092024-02-15T03:05:48.578Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459141492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184102024-02-15T03:05:48.578Z{"msg":"Wait for a query_work_queue command to finish before sending IO
184112024-02-15T03:05:48.578Z[1]R flush_numbers[0..12]: [259, 259, 267, 271, 271, 271, 267, 262, 271, 271, 268, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459171658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184122024-02-15T03:05:48.578Z{"msg":"[1]R generation[0..12]: [158, 158, 164, 165, 165, 165, 164, 160, 165, 165, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459190905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184132024-02-15T03:05:48.578Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459217732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184142024-02-15T03:05:48.578Z{"msg":"[2]R flush_numbers[0..12]: [259, 259, 267, 271, 271, 271, 267, 262, 271, 271, 268, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.45923573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184152024-02-15T03:05:48.578Z{"msg":"[2]R generation[0..12]: [158, 158, 164, 165, 165, 165, 164, 160, 165, 165, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459264026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184162024-02-15T03:05:48.578Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459283304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184172024-02-15T03:05:48.578Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459308041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184182024-02-15T03:05:48.578Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459326419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184192024-02-15T03:05:48.578Z{"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459364654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184202024-02-15T03:05:48.578Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459384561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"":"downstairs","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184212024-02-15T03:05:48.578Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459428606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184222024-02-15T03:05:48.578Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459468951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"0","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184232024-02-15T03:05:48.578Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459492698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"1","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184242024-02-15T03:05:48.578Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459529983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"client":"2","":"downstairs","session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184252024-02-15T03:05:48.578Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459571138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184262024-02-15T03:05:48.578Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459601684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184272024-02-15T03:05:48.578Z{"msg":"c236053e-4460-433e-ae01-5e332f808347 is now active with session: 3e126cc2-641f-4497-8c71-ba7036bd31d9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459641539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184282024-02-15T03:05:48.578Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459678735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808,"session_id":"3e126cc2-641f-4497-8c71-ba7036bd31d9"}
184292024-02-15T03:05:48.578Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:47.459718899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808}
184302024-02-15T03:05:48.579ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184312024-02-15T03:05:48.579ZLoading write count information from file "/tmp/test_repair_verify.data"
184322024-02-15T03:05:48.579ZRun Repair workload
184332024-02-15T03:05:48.579Z01/30 Write block 436 len 4096 data: 12
184342024-02-15T03:05:48.591Z02/30 Read block 220 len 12288
184352024-02-15T03:05:48.591Z03/30 Write block 475 len 12288 data: 9 9 10
184362024-02-15T03:05:48.591Z04/30 Write block 188 len 8192 data: 15 18
184372024-02-15T03:05:48.591Z05/30 Read block 47 len 28672
184382024-02-15T03:05:48.592Z06/30 Flush
184392024-02-15T03:05:48.592Z07/30 Write block 396 len 16384 data: 13 13 13 15
184402024-02-15T03:05:48.592Z08/30 Write block 261 len 20480 data: 14 10 9 9 11
184412024-02-15T03:05:48.592Z09/30 Write block 313 len 8192 data: 12 12
184422024-02-15T03:05:48.592Z10/30 Read block 89 len 20480
184432024-02-15T03:05:48.593Z11/30 Read block 322 len 4096
184442024-02-15T03:05:48.593Z12/30 Read block 62 len 16384
184452024-02-15T03:05:48.593Z13/30 Flush
184462024-02-15T03:05:48.597Z14/30 Write block 390 len 36864 data: 14 13 12 11 12 12 14 14 14
184472024-02-15T03:05:48.597Z15/30 Read block 502 len 36864
184482024-02-15T03:05:48.598Z16/30 Read block 337 len 36864
184492024-02-15T03:05:48.598Z17/30 Write block 255 len 28672 data: 12 13 16 16 20 17 15
184502024-02-15T03:05:48.598Z18/30 Read block 460 len 32768
184512024-02-15T03:05:48.598Z19/30 Write block 74 len 12288 data: 8 11 11
184522024-02-15T03:05:48.598Z20/30 Read block 365 len 36864
184532024-02-15T03:05:48.598Z21/30 Read block 309 len 40960
184542024-02-15T03:05:48.598Z22/30 Write block 166 len 28672 data: 14 12 11 10 8 9 7
184552024-02-15T03:05:48.598Z23/30 Write block 362 len 32768 data: 13 9 11 11 12 13 10 10
184562024-02-15T03:05:48.608Z24/30 Write block 484 len 40960 data: 13 11 14 15 11 14 16 13 15 11
184572024-02-15T03:05:48.609Z25/30 Write block 57 len 40960 data: 15 15 12 13 12 13 14 15 13 12
184582024-02-15T03:05:48.609Z26/30 Flush
184592024-02-15T03:05:48.612Z27/30 Write block 134 len 16384 data: 13 12 11 13
184602024-02-15T03:05:48.612Z28/30 Read block 155 len 36864
184612024-02-15T03:05:48.613Z29/30 Write block 256 len 40960 data: 14 17 17 21 18 16 11 10 10 12
184622024-02-15T03:05:48.613Z30/30 Write block 409 len 16384 data: 19 21 21 19
184632024-02-15T03:05:48.613Z----------------------------------------------------------------
184642024-02-15T03:05:48.613Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:24
184652024-02-15T03:05:48.613ZStates: Active Active Active
184662024-02-15T03:05:48.613ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
184672024-02-15T03:05:48.613Z 7 Acked 1006 Write 4 Done Done Done false
184682024-02-15T03:05:48.613Z 8 Acked 1007 Write 5 Done Done Done false
184692024-02-15T03:05:48.613Z 9 Acked 1008 Write 2 Done Done Done false
184702024-02-15T03:05:48.613Z 10 Acked 1009 Read 5 Done Done Done false
184712024-02-15T03:05:48.613Z 11 Acked 1010 Read 1 Done Done Done false
184722024-02-15T03:05:48.613Z 12 Acked 1011 Read 4 Sent Done Done false
184732024-02-15T03:05:48.613Z 13 Acked 1012 Flush 0 Sent Done Done false
184742024-02-15T03:05:48.613Z 14 Acked 1013 Write 9 Sent Done Done false
184752024-02-15T03:05:48.613Z 15 Acked 1014 Read 9 Sent Done Done false
184762024-02-15T03:05:48.613Z 16 Acked 1015 Read 9 Sent Done Done false
184772024-02-15T03:05:48.613Z 17 Acked 1016 Write 7 Sent Done Done false
184782024-02-15T03:05:48.613Z 18 Acked 1017 Read 8 Sent Done Done false
184792024-02-15T03:05:48.613Z 19 Acked 1018 Write 3 Sent Done Done false
184802024-02-15T03:05:48.613Z 20 Acked 1019 Read 9 Sent Done Done false
184812024-02-15T03:05:48.614Z 21 Acked 1020 Read 10 Sent Done Done false
184822024-02-15T03:05:48.614Z 22 Acked 1021 Write 7 Sent Done Done false
184832024-02-15T03:05:48.614Z 23 Acked 1022 Write 8 Sent Done Done false
184842024-02-15T03:05:48.614Z 24 Acked 1023 Write 10 Sent Done Done false
184852024-02-15T03:05:48.614Z 25 Acked 1024 Write 10 Sent Done Done false
184862024-02-15T03:05:48.614Z 26 Acked 1025 Flush 0 Sent Done Done false
184872024-02-15T03:05:48.614Z 27 Acked 1026 Write 4 Sent Done Done false
184882024-02-15T03:05:48.614Z 28 Acked 1027 Read 9 Sent Done Done false
184892024-02-15T03:05:48.614Z 29 Acked 1028 Write 10 Sent Sent Sent false
184902024-02-15T03:05:48.614Z 30 Acked 1029 Write 4 Sent Sent Sent false
184912024-02-15T03:05:48.614Z STATES DS:0 DS:1 DS:2 TOTAL
184922024-02-15T03:05:48.614Z New 0 0 0 0
184932024-02-15T03:05:48.614Z Sent 19 2 2 23
184942024-02-15T03:05:48.614Z Done 5 22 22 49
184952024-02-15T03:05:48.614Z Skipped 0 0 0 0
184962024-02-15T03:05:48.614Z Error 0 0 0 0
184972024-02-15T03:05:48.614ZLast Flush: 1005 1025 1025
184982024-02-15T03:05:48.614ZDownstairs last five completed: 1005 1004 1003 1002 1001
184992024-02-15T03:05:48.614ZUpstairs last five completed: 30 29 28 27 26
185002024-02-15T03:05:48.614Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:47.494718274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1808}
185012024-02-15T03:05:48.614ZWrote out file "/tmp/test_repair_verify.data"
185022024-02-15T03:05:48.618Z
185032024-02-15T03:05:48.618ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185042024-02-15T03:05:48.636ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185052024-02-15T03:05:48.636Z 0 000-019 158 158 158 259 259 259 F F F
185062024-02-15T03:05:48.636Z 1 020-039 158 158 158 259 259 259 F F F
185072024-02-15T03:05:48.637Z 2 040-059 164 166 166 267 274 274 F F F <---
185082024-02-15T03:05:48.637Z 3 060-079 165 166 166 271 274 274 F F F <---
185092024-02-15T03:05:48.637Z 4 080-099 165 165 165 271 271 271 F F F
185102024-02-15T03:05:48.637Z 5 100-119 165 165 165 271 271 271 F F F
185112024-02-15T03:05:48.637Z 6 120-139 164 164 164 267 267 267 F T T <---
185122024-02-15T03:05:48.637Z 7 140-159 160 160 160 262 262 262 F F F
185132024-02-15T03:05:48.637Z 8 160-179 165 166 166 271 274 274 F F F <---
185142024-02-15T03:05:48.637Z 9 180-199 166 166 166 272 272 272 F F F
185152024-02-15T03:05:48.637Z 10 200-219 164 164 164 268 268 268 F F F
185162024-02-15T03:05:48.637Z 11 220-239 160 160 160 262 262 262 F F F
185172024-02-15T03:05:48.637Z 12 240-259 164 166 166 269 274 274 F T T <---
185182024-02-15T03:05:48.637Z 13 260-279 162 166 166 264 274 274 T T T <---
185192024-02-15T03:05:48.637Z 14 280-299 165 165 165 271 271 271 F F F
185202024-02-15T03:05:48.637Z 15 300-319 161 166 166 263 273 273 T F F <---
185212024-02-15T03:05:48.637Z 16 320-339 161 161 161 263 263 263 F F F
185222024-02-15T03:05:48.637Z 17 340-359 159 159 159 260 260 260 F F F
185232024-02-15T03:05:48.637Z 18 360-379 164 166 166 266 274 274 F F F <---
185242024-02-15T03:05:48.637Z 19 380-399 164 166 166 266 274 274 T F F <---
185252024-02-15T03:05:48.637Z 20 400-419 163 163 163 265 265 265 F T T <---
185262024-02-15T03:05:48.637Z 21 420-439 166 166 166 272 272 272 F F F
185272024-02-15T03:05:48.637Z 22 440-459 165 165 165 271 271 271 F F F
185282024-02-15T03:05:48.642Z 23 460-479 166 166 166 272 272 272 F F F
185292024-02-15T03:05:48.642Z 24 480-499 160 166 166 262 274 274 F F F <---
185302024-02-15T03:05:48.642Z 25 500-519 165 165 165 271 271 271 F F F
185312024-02-15T03:05:48.642Z 26 520-539 164 164 164 269 269 269 F F F
185322024-02-15T03:05:48.642Z 27 540-559 159 159 159 260 260 260 F F F
185332024-02-15T03:05:48.642Z 28 560-579 164 164 164 270 270 270 F F F
185342024-02-15T03:05:48.642Z 29 580-599 164 164 164 270 270 270 F F F
185352024-02-15T03:05:48.642ZMax gen: 166, Max flush: 274
185362024-02-15T03:05:48.642ZError: Difference in extent metadata found!
185372024-02-15T03:05:48.642ZA Difference in extent metadata is expected here
185382024-02-15T03:05:48.642ZOn loop 83
185392024-02-15T03:05:48.642Z
185402024-02-15T03:05:48.645ZVerifying data now
185412024-02-15T03:05:54.259ZNo change after verify
185422024-02-15T03:05:54.259ZLoop: 83 Downstairs dump after verify (and repair):
185432024-02-15T03:05:54.301ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185442024-02-15T03:05:54.301Z 0 000-019 158 158 158 259 259 259 F F F
185452024-02-15T03:05:54.301Z 1 020-039 158 158 158 259 259 259 F F F
185462024-02-15T03:05:54.301Z 2 040-059 166 166 166 274 274 274 F F F
185472024-02-15T03:05:54.301Z 3 060-079 166 166 166 274 274 274 F F F
185482024-02-15T03:05:54.301Z 4 080-099 165 165 165 271 271 271 F F F
185492024-02-15T03:05:54.302Z 5 100-119 165 165 165 271 271 271 F F F
185502024-02-15T03:05:54.303Z 6 120-139 167 167 167 275 275 275 F F F
185512024-02-15T03:05:54.303Z 7 140-159 160 160 160 262 262 262 F F F
185522024-02-15T03:05:54.303Z 8 160-179 166 166 166 274 274 274 F F F
185532024-02-15T03:05:54.303Z 9 180-199 166 166 166 272 272 272 F F F
185542024-02-15T03:05:54.303Z 10 200-219 164 164 164 268 268 268 F F F
185552024-02-15T03:05:54.303Z 11 220-239 160 160 160 262 262 262 F F F
185562024-02-15T03:05:54.303Z 12 240-259 167 167 167 275 275 275 F F F
185572024-02-15T03:05:54.303Z 13 260-279 167 167 167 275 275 275 F F F
185582024-02-15T03:05:54.304Z 14 280-299 165 165 165 271 271 271 F F F
185592024-02-15T03:05:54.304Z 15 300-319 166 166 166 273 273 273 F F F
185602024-02-15T03:05:54.304Z 16 320-339 161 161 161 263 263 263 F F F
185612024-02-15T03:05:54.304Z 17 340-359 159 159 159 260 260 260 F F F
185622024-02-15T03:05:54.304Z 18 360-379 166 166 166 274 274 274 F F F
185632024-02-15T03:05:54.304Z 19 380-399 166 166 166 274 274 274 F F F
185642024-02-15T03:05:54.304Z 20 400-419 167 167 167 275 275 275 F F F
185652024-02-15T03:05:54.304Z 21 420-439 166 166 166 272 272 272 F F F
185662024-02-15T03:05:54.304Z 22 440-459 165 165 165 271 271 271 F F F
185672024-02-15T03:05:54.304Z 23 460-479 166 166 166 272 272 272 F F F
185682024-02-15T03:05:54.305Z 24 480-499 166 166 166 274 274 274 F F F
185692024-02-15T03:05:54.305Z 25 500-519 165 165 165 271 271 271 F F F
185702024-02-15T03:05:54.305Z 26 520-539 164 164 164 269 269 269 F F F
185712024-02-15T03:05:54.305Z 27 540-559 159 159 159 260 260 260 F F F
185722024-02-15T03:05:54.305Z 28 560-579 164 164 164 270 270 270 F F F
185732024-02-15T03:05:54.305Z 29 580-599 164 164 164 270 270 270 F F F
185742024-02-15T03:05:54.305ZMax gen: 167, Max flush: 275
185752024-02-15T03:05:54.305Z
185762024-02-15T03:05:54.305ZBegin loop 84
185772024-02-15T03:05:54.305ZDownstairs to restart: 0
185782024-02-15T03:05:54.306Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.127456129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816}
185792024-02-15T03:05:54.306Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.127728565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816}
185802024-02-15T03:05:54.306Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.127754801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816}
185812024-02-15T03:05:54.306Z{"msg":"Crucible ce964124-cdb4-4f51-9993-a7933636a7ff has session id: 036dc673-2b7f-4794-9cb0-ea9c3a28dec0","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.127784567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816Crucible runtime is spawned
185822024-02-15T03:05:54.306Z,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185832024-02-15T03:05:54.306Z{"msg":"Upstairs opts: Upstairs UUID: ce964124-cdb4-4f51-9993-a7933636a7ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.12784576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185842024-02-15T03:05:54.306Z{"msg":"Crucible stats registered with UUID: ce964124-cdb4-4f51-9993-a7933636a7ff","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.127884695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185852024-02-15T03:05:54.306Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.127908692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816}
185862024-02-15T03:05:54.306Z{"msg":"ce964124-cdb4-4f51-9993-a7933636a7ff active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128024728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185872024-02-15T03:05:54.307Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128078531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185882024-02-15T03:05:54.307Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128113826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185892024-02-15T03:05:54.307Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128150852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185902024-02-15T03:05:54.307Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128182468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185912024-02-15T03:05:54.307Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128209944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185922024-02-15T03:05:54.307Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128234781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185932024-02-15T03:05:54.307Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128275566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185942024-02-15T03:05:54.307Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.12832316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185952024-02-15T03:05:54.307Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128365075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185962024-02-15T03:05:54.307Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128392881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185972024-02-15T03:05:54.307Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128428427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185982024-02-15T03:05:54.307Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:53.128648829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
185992024-02-15T03:05:54.308Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:53.128690184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186002024-02-15T03:05:54.308Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:53.128709532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186012024-02-15T03:05:54.308Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.12872266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186022024-02-15T03:05:54.308Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128737308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186032024-02-15T03:05:54.308Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.128812599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186042024-02-15T03:05:54.308Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.129195851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186052024-02-15T03:05:54.309Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.129235476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186062024-02-15T03:05:54.309Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.129261273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186072024-02-15T03:05:54.309Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.129301177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186082024-02-15T03:05:54.309Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.129779837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186092024-02-15T03:05:54.309Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.12992222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186102024-02-15T03:05:54.309Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.130337858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186112024-02-15T03:05:54.309Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.130376903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186122024-02-15T03:05:54.309Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.130403889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186132024-02-15T03:05:54.309Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.130428036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186142024-02-15T03:05:54.309Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.130442254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186152024-02-15T03:05:54.309Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:53.130462412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186162024-02-15T03:05:55.246Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.130200998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186172024-02-15T03:05:55.253Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.130363937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186182024-02-15T03:05:55.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.131194093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186192024-02-15T03:05:55.253Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.131234998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186202024-02-15T03:05:55.253Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.131783449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186212024-02-15T03:05:55.253Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132392893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186222024-02-15T03:05:55.253Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132464344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186232024-02-15T03:05:55.253Z{"msg":"[0]R flush_numbers[0..12]: [259, 259, 274, 274, 271, 271, 275, 262, 274, 272, 268, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132520617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186242024-02-15T03:05:55.253Z{"msg":"[0]R generation[0..12]: [158, 158, 166, 166, 165, 165, 167, 160, 166, 166, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132553873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186252024-02-15T03:05:55.253Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132570561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186262024-02-15T03:05:55.253Z{"msg":"[1]R flush_numbers[0..12]: [259, 259, 274, 274, 271, 271, 275, 262, 274, 272, 268, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132593478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186272024-02-15T03:05:55.253Z{"msg":"[1]R generation[0..12]: [158, 158, 166, 166, 165, 165, 167, 160, 166, 166, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132608846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186282024-02-15T03:05:55.253Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
186292024-02-15T03:05:55.253Z,"time":"2024-02-15T03:05:54.132630603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186302024-02-15T03:05:55.253Z{"msg":"[2]R flush_numbers[0..12]: [259, 259, 274, 274, 271, 271, 275, 262, 274, 272, 268, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132650251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186312024-02-15T03:05:55.253Z{"msg":"[2]R generation[0..12]: [158, 158, 166, 166, 165, 165, 167, 160, 166, 166, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132672388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186322024-02-15T03:05:55.254Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132687166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186332024-02-15T03:05:55.254Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132706374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186342024-02-15T03:05:55.254Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132720472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186352024-02-15T03:05:55.254Z{"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132744079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186362024-02-15T03:05:55.254Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132764366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"":"downstairs","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186372024-02-15T03:05:55.254Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132779455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186382024-02-15T03:05:55.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132800912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"0","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186392024-02-15T03:05:55.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.13281582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"1","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186402024-02-15T03:05:55.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132830538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"client":"2","":"downstairs","session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186412024-02-15T03:05:55.254Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132852146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186422024-02-15T03:05:55.254Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132866004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186432024-02-15T03:05:55.254Z{"msg":"ce964124-cdb4-4f51-9993-a7933636a7ff is now active with session: 036dc673-2b7f-4794-9cb0-ea9c3a28dec0","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132879522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186442024-02-15T03:05:55.254Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132891671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816,"session_id":"036dc673-2b7f-4794-9cb0-ea9c3a28dec0"}
186452024-02-15T03:05:55.254Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:54.132911408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816}
186462024-02-15T03:05:55.254ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186472024-02-15T03:05:55.254ZLoading write count information from file "/tmp/test_repair_verify.data"
186482024-02-15T03:05:55.255ZRun Repair workload
186492024-02-15T03:05:55.255Z01/30 Write block 221 len 40960 data: 15 15 13 12 12 12 10 12 10 9
186502024-02-15T03:05:55.260Z02/30 Write block 63 len 24576 data: 15 16 14 13 14 15
186512024-02-15T03:05:55.260Z03/30 Read block 125 len 40960
186522024-02-15T03:05:55.260Z04/30 Read block 376 len 8192
186532024-02-15T03:05:55.260Z05/30 Read block 469 len 4096
186542024-02-15T03:05:55.260Z06/30 Read block 500 len 8192
186552024-02-15T03:05:55.260Z07/30 Read block 528 len 36864
186562024-02-15T03:05:55.260Z08/30 Read block 392 len 32768
186572024-02-15T03:05:55.260Z09/30 Write block 0 len 4096 data: 2
186582024-02-15T03:05:55.260Z10/30 Write block 424 len 8192 data: 11 11
186592024-02-15T03:05:55.260Z11/30 Read block 100 len 24576
186602024-02-15T03:05:55.261Z12/30 Write block 226 len 20480 data: 13 11 13 11 10
186612024-02-15T03:05:55.275Z13/30 Write block 589 len 20480 data: 14 15 17 14 14
186622024-02-15T03:05:55.275Z14/30 Write block 406 len 40960 data: 13 15 15 20 22 22 20 20 16 12
186632024-02-15T03:05:55.275Z15/30 Write block 409 len 20480 data: 21 23 23 21 21
186642024-02-15T03:05:55.275Z16/30 Read block 73 len 28672
186652024-02-15T03:05:55.275Z17/30 Read block 255 len 4096
186662024-02-15T03:05:55.275Z18/30 Write block 517 len 36864 data: 13 13 12 10 8 7 9 7 6
186672024-02-15T03:05:55.275Z19/30 Flush
186682024-02-15T03:05:55.275Z20/30 Write block 179 len 28672 data: 13 13 12 10 13 15 16
186692024-02-15T03:05:55.275Z21/30 Write block 362 len 20480 data: 14 10 12 12 13
186702024-02-15T03:05:55.275Z22/30 Write block 27 len 32768 data: 10 12 11 13 14 13 16 16
186712024-02-15T03:05:55.281Z23/30 Read block 458 len 36864
186722024-02-15T03:05:55.281Z24/30 Read block 126 len 40960
186732024-02-15T03:05:55.281Z25/30 Write block 507 len 40960 data: 10 10 15 14 12 14 11 14 13 14
186742024-02-15T03:05:55.281Z26/30 Read block 358 len 24576
186752024-02-15T03:05:55.281Z27/30 Read block 373 len 4096
186762024-02-15T03:05:55.281Z28/30 Read block 360 len 20480
186772024-02-15T03:05:55.281Z29/30 Read block 526 len 24576
186782024-02-15T03:05:55.281Z30/30 Read block 462 len 4096
186792024-02-15T03:05:55.281Z----------------------------------------------------------------
186802024-02-15T03:05:55.281Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
186812024-02-15T03:05:55.281ZStates: Active Active Active
186822024-02-15T03:05:55.282ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
186832024-02-15T03:05:55.282Z 1 Acked 1000 Write 10 Sent Done Done false
186842024-02-15T03:05:55.282Z 2 Acked 1001 Write 6 Sent Done Done false
186852024-02-15T03:05:55.282Z 3 Acked 1002 Read 10 Sent Done Done false
186862024-02-15T03:05:55.282Z 4 Acked 1003 Read 2 Sent Done Done false
186872024-02-15T03:05:55.282Z 5 Acked 1004 Read 1 Sent Done Done false
186882024-02-15T03:05:55.282Z 6 Acked 1005 Read 2 Sent Done Done false
186892024-02-15T03:05:55.282Z 7 Acked 1006 Read 9 Sent Done Done false
186902024-02-15T03:05:55.282Z 8 Acked 1007 Read 8 Sent Done Done false
186912024-02-15T03:05:55.282Z 9 Acked 1008 Write 1 Sent Done Done false
186922024-02-15T03:05:55.282Z 10 Acked 1009 Write 2 Sent Done Done false
186932024-02-15T03:05:55.282Z 11 Acked 1010 Read 6 Sent Done Done false
186942024-02-15T03:05:55.282Z 12 Acked 1011 Write 5 Sent Done Done false
186952024-02-15T03:05:55.282Z 13 Acked 1012 Write 5 Sent Done Done false
186962024-02-15T03:05:55.282Z 14 Acked 1013 Write 10 Sent Done Done false
186972024-02-15T03:05:55.282Z 15 Acked 1014 Write 5 Sent Done Done false
186982024-02-15T03:05:55.282Z 16 Acked 1015 Read 7 Sent Done Done false
186992024-02-15T03:05:55.282Z 17 Acked 1016 Read 1 Sent Done Done false
187002024-02-15T03:05:55.282Z 18 Acked 1017 Write 9 Sent Done Done false
187012024-02-15T03:05:55.282Z 19 Acked 1018 Flush 0 Sent Done Done false
187022024-02-15T03:05:55.282Z 20 Acked 1019 Write 7 Sent Done Done false
187032024-02-15T03:05:55.282Z 21 Acked 1020 Write 5 Sent Done Done false
187042024-02-15T03:05:55.282Z 22 Acked 1021 Write 8 Sent Done Done false
187052024-02-15T03:05:55.283Z 23 Acked 1022 Read 9 Sent Done Done false
187062024-02-15T03:05:55.283Z 24 Acked 1023 Read 10 Sent Done Done false
187072024-02-15T03:05:55.283Z 25 Acked 1024 Write 10 Sent Done Done false
187082024-02-15T03:05:55.283Z 26 Acked 1025 Read 6 Sent Done Done false
187092024-02-15T03:05:55.283Z 27 Acked 1026 Read 1 Sent Done Done false
187102024-02-15T03:05:55.283Z 28 Acked 1027 Read 5 Sent Done Done false
187112024-02-15T03:05:55.283Z 29 Acked 1028 Read 6 Sent Done Done false
187122024-02-15T03:05:55.283Z 30 Acked 1029 Read 1 Sent Sent Done false
187132024-02-15T03:05:55.283Z STATES DS:0 DS:1 DS:2 TOTAL
187142024-02-15T03:05:55.283Z New 0 0 0 0
187152024-02-15T03:05:55.283Z Sent 30 1 0 31
187162024-02-15T03:05:55.283Z Done 0 29 30 59
187172024-02-15T03:05:55.283Z Skipped 0 0 0 0
187182024-02-15T03:05:55.283Z Error 0 0 0 0
187192024-02-15T03:05:55.283ZLast Flush: 0 1018 1018
187202024-02-15T03:05:55.283ZDownstairs last five completed:
187212024-02-15T03:05:55.283ZUpstairs last five completed: 30 29 28 27 26
187222024-02-15T03:05:55.283Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:54.163650367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1816}
187232024-02-15T03:05:55.283ZWrote out file "/tmp/test_repair_verify.data"
187242024-02-15T03:05:55.292Z
187252024-02-15T03:05:55.292ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187262024-02-15T03:05:55.306ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187272024-02-15T03:05:55.306Z 0 000-019 158 168 168 259 276 276 F F F <---
187282024-02-15T03:05:55.306Z 1 020-039 158 158 158 259 259 259 F T T <---
187292024-02-15T03:05:55.306Z 2 040-059 166 166 166 274 274 274 F F F
187302024-02-15T03:05:55.307Z 3 060-079 166 168 168 274 276 276 F F F <---
187312024-02-15T03:05:55.307Z 4 080-099 165 165 165 271 271 271 F F F
187322024-02-15T03:05:55.307Z 5 100-119 165 165 165 271 271 271 F F F
187332024-02-15T03:05:55.307Z 6 120-139 167 167 167 275 275 275 F F F
187342024-02-15T03:05:55.307Z 7 140-159 160 160 160 262 262 262 F F F
187352024-02-15T03:05:55.312Z 8 160-179 166 166 166 274 274 274 F T T <---
187362024-02-15T03:05:55.312Z 9 180-199 166 166 166 272 272 272 F T T <---
187372024-02-15T03:05:55.312Z 10 200-219 164 164 164 268 268 268 F F F
187382024-02-15T03:05:55.312Z 11 220-239 160 168 168 262 276 276 F F F <---
187392024-02-15T03:05:55.312Z 12 240-259 167 167 167 275 275 275 F F F
187402024-02-15T03:05:55.312Z 13 260-279 167 167 167 275 275 275 F F F
187412024-02-15T03:05:55.313Z 14 280-299 165 165 165 271 271 271 F F F
187422024-02-15T03:05:55.313Z 15 300-319 166 166 166 273 273 273 F F F
187432024-02-15T03:05:55.313Z 16 320-339 161 161 161 263 263 263 F F F
187442024-02-15T03:05:55.313Z 17 340-359 159 159 159 260 260 260 F F F
187452024-02-15T03:05:55.313Z 18 360-379 166 166 166 274 274 274 F T T <---
187462024-02-15T03:05:55.314Z 19 380-399 166 166 166 274 274 274 F F F
187472024-02-15T03:05:55.314Z 20 400-419 167 168 168 275 276 276 F F F <---
187482024-02-15T03:05:55.314Z 21 420-439 166 168 168 272 276 276 F F F <---
187492024-02-15T03:05:55.314Z 22 440-459 165 165 165 271 271 271 F F F
187502024-02-15T03:05:55.314Z 23 460-479 166 166 166 272 272 272 F F F
187512024-02-15T03:05:55.314Z 24 480-499 166 166 166 274 274 274 F F F
187522024-02-15T03:05:55.314Z 25 500-519 165 168 168 271 276 276 F T T <---
187532024-02-15T03:05:55.314Z 26 520-539 164 168 168 269 276 276 F F F <---
187542024-02-15T03:05:55.314Z 27 540-559 159 159 159 260 260 260 F F F
187552024-02-15T03:05:55.314Z 28 560-579 164 164 164 270 270 270 F F F
187562024-02-15T03:05:55.314Z 29 580-599 164 168 168 270 276 276 F F F <---
187572024-02-15T03:05:55.314ZMax gen: 168, Max flush: 276
187582024-02-15T03:05:55.314ZError: Difference in extent metadata found!
187592024-02-15T03:05:55.314ZA Difference in extent metadata is expected here
187602024-02-15T03:05:55.314ZOn loop 84
187612024-02-15T03:05:55.314Z
187622024-02-15T03:05:55.354ZVerifying data now
187632024-02-15T03:05:59.931ZNo change after verify
187642024-02-15T03:05:59.931ZLoop: 84 Downstairs dump after verify (and repair):
187652024-02-15T03:05:59.946ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187662024-02-15T03:05:59.947Z 0 000-019 168 168 168 276 276 276 F F F
187672024-02-15T03:05:59.947Z 1 020-039 169 169 169 277 277 277 F F F
187682024-02-15T03:05:59.947Z 2 040-059 166 166 166 274 274 274 F F F
187692024-02-15T03:05:59.947Z 3 060-079 168 168 168 276 276 276 F F F
187702024-02-15T03:05:59.947Z 4 080-099 165 165 165 271 271 271 F F F
187712024-02-15T03:05:59.947Z 5 100-119 165 165 165 271 271 271 F F F
187722024-02-15T03:05:59.947Z 6 120-139 167 167 167 275 275 275 F F F
187732024-02-15T03:05:59.952Z 7 140-159 160 160 160 262 262 262 F F F
187742024-02-15T03:05:59.952Z 8 160-179 169 169 169 277 277 277 F F F
187752024-02-15T03:05:59.952Z 9 180-199 169 169 169 277 277 277 F F F
187762024-02-15T03:05:59.952Z 10 200-219 164 164 164 268 268 268 F F F
187772024-02-15T03:05:59.952Z 11 220-239 168 168 168 276 276 276 F F F
187782024-02-15T03:05:59.952Z 12 240-259 167 167 167 275 275 275 F F F
187792024-02-15T03:05:59.952Z 13 260-279 167 167 167 275 275 275 F F F
187802024-02-15T03:05:59.952Z 14 280-299 165 165 165 271 271 271 F F F
187812024-02-15T03:05:59.953Z 15 300-319 166 166 166 273 273 273 F F F
187822024-02-15T03:05:59.953Z 16 320-339 161 161 161 263 263 263 F F F
187832024-02-15T03:05:59.953Z 17 340-359 159 159 159 260 260 260 F F F
187842024-02-15T03:05:59.953Z 18 360-379 169 169 169 277 277 277 F F F
187852024-02-15T03:05:59.953Z 19 380-399 166 166 166 274 274 274 F F F
187862024-02-15T03:05:59.953Z 20 400-419 168 168 168 276 276 276 F F F
187872024-02-15T03:05:59.953Z 21 420-439 168 168 168 276 276 276 F F F
187882024-02-15T03:05:59.953Z 22 440-459 165 165 165 271 271 271 F F F
187892024-02-15T03:05:59.954Z 23 460-479 166 166 166 272 272 272 F F F
187902024-02-15T03:05:59.954Z 24 480-499 166 166 166 274 274 274 F F F
187912024-02-15T03:05:59.954Z 25 500-519 169 169 169 277 277 277 F F F
187922024-02-15T03:05:59.954Z 26 520-539 168 168 168 276 276 276 F F F
187932024-02-15T03:05:59.954Z 27 540-559 159 159 159 260 260 260 F F F
187942024-02-15T03:05:59.954Z 28 560-579 164 164 164 270 270 270 F F F
187952024-02-15T03:05:59.954Z 29 580-599 168 168 168 276 276 276 F F F
187962024-02-15T03:05:59.954ZMax gen: 169, Max flush: 277
187972024-02-15T03:05:59.955Z
187982024-02-15T03:05:59.955ZBegin loop 85
187992024-02-15T03:05:59.955ZDownstairs to restart: 2
188002024-02-15T03:05:59.970Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.853929059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824}
188012024-02-15T03:05:59.970Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854325119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"Crucible runtime is spawned
188022024-02-15T03:05:59.971Z:1824}
188032024-02-15T03:05:59.977Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854377633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824}
188042024-02-15T03:05:59.977Z{"msg":"Crucible bb823e07-ad66-4266-80d2-6f81d2cb30f7 has session id: 371303f8-b893-4041-9e25-4494f14afd02","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854432226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188052024-02-15T03:05:59.977Z{"msg":"Upstairs opts: Upstairs UUID: bb823e07-ad66-4266-80d2-6f81d2cb30f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854492278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188062024-02-15T03:05:59.977Z{"msg":"Crucible stats registered with UUID: bb823e07-ad66-4266-80d2-6f81d2cb30f7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854534593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188072024-02-15T03:05:59.977Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.85456108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824}
188082024-02-15T03:05:59.977Z{"msg":"bb823e07-ad66-4266-80d2-6f81d2cb30f7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854601474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188092024-02-15T03:05:59.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854624952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188102024-02-15T03:05:59.978Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854665517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188112024-02-15T03:05:59.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854690283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188122024-02-15T03:05:59.978Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.85471911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188132024-02-15T03:05:59.978Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854736798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188142024-02-15T03:05:59.978Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854756135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188152024-02-15T03:05:59.978Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854776803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188162024-02-15T03:05:59.978Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854851053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188172024-02-15T03:05:59.978Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.854961209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188182024-02-15T03:05:59.979Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855056468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188192024-02-15T03:05:59.979Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855102042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188202024-02-15T03:05:59.979Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:58.855157405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188212024-02-15T03:05:59.979Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:58.855203569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188222024-02-15T03:05:59.979Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:58.855247864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188232024-02-15T03:05:59.979Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.85527522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188242024-02-15T03:05:59.979Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855302237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188252024-02-15T03:05:59.980Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855326704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188262024-02-15T03:05:59.980Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855636485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188272024-02-15T03:05:59.980Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855688548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188282024-02-15T03:05:59.980Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855716875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188292024-02-15T03:05:59.980Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.855741532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188302024-02-15T03:05:59.980Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856277795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188312024-02-15T03:05:59.980Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856371473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188322024-02-15T03:05:59.980Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856785001Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188332024-02-15T03:05:59.980Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856824616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188342024-02-15T03:05:59.980Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856861671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188352024-02-15T03:05:59.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856885738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188362024-02-15T03:05:59.981Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856915595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188372024-02-15T03:05:59.981Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:58.856939901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188382024-02-15T03:06:00.973Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.856637342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188392024-02-15T03:06:00.973Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.856757247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188402024-02-15T03:06:00.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.857676622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188412024-02-15T03:06:00.984Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.857723986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188422024-02-15T03:06:00.984Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.858270777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188432024-02-15T03:06:00.984Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.858926015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188442024-02-15T03:06:00.984Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.858981768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188452024-02-15T03:06:00.984Z{"msg":"[0]R flush_numbers[0..12]: [276, 277, 274, 276, 271, 271, 275, 262, 277, 277, 268, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859019994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188462024-02-15T03:06:00.984Z{"msg":"[0]R generation[0..12]: [168, 169, 166, 168, 165, 165, 167, 160, 169, 169, 164, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859068437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188472024-02-15T03:06:00.984Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859097004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188482024-02-15T03:06:00.984Z{"msg":"[1]R flush_numbers[0..12]: [276, 277, 274, 276, 271, 271, 275, 262, 277, 277, 268, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859116482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188492024-02-15T03:06:00.984Z{"msg":"[1]R generation[0..12]: [168, 169, 166, 168, 165, 165, 167, 160, 169, 169, 164, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859145448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188502024-02-15T03:06:00.984Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
188512024-02-15T03:06:00.984Z,"time":"2024-02-15T03:05:59.859188273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188522024-02-15T03:06:00.984Z{"msg":"[2]R flush_numbers[0..12]: [276, 277, 274, 276, 271, 271, 275, 262, 277, 277, 268, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859235367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188532024-02-15T03:06:00.984Z{"msg":"[2]R generation[0..12]: [168, 169, 166, 168, 165, 165, 167, 160, 169, 169, 164, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859270342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188542024-02-15T03:06:00.984Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.85929041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188552024-02-15T03:06:00.984Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859333015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188562024-02-15T03:06:00.984Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.85936806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188572024-02-15T03:06:00.985Z{"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859386938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188582024-02-15T03:06:00.985Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859413234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"":"downstairs","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188592024-02-15T03:06:00.985Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.85945131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188602024-02-15T03:06:00.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859483705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"0","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188612024-02-15T03:06:00.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859499783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"1","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188622024-02-15T03:06:00.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.85953129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"client":"2","":"downstairs","session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188632024-02-15T03:06:00.985Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859567475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188642024-02-15T03:06:00.985Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859593302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188652024-02-15T03:06:00.985Z{"msg":"bb823e07-ad66-4266-80d2-6f81d2cb30f7 is now active with session: 371303f8-b893-4041-9e25-4494f14afd02","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.8596108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188662024-02-15T03:06:00.985Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859644196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824,"session_id":"371303f8-b893-4041-9e25-4494f14afd02"}
188672024-02-15T03:06:00.985Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
188682024-02-15T03:06:00.985ZThe guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:05:59.859683831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824}
188692024-02-15T03:06:00.985ZLoading write count information from file "/tmp/test_repair_verify.data"
188702024-02-15T03:06:00.985ZRun Repair workload
188712024-02-15T03:06:00.985Z01/30 Write block 156 len 28672 data: 18 18 14 11 9 11 14
188722024-02-15T03:06:00.985Z02/30 Read block 560 len 40960
188732024-02-15T03:06:00.992Z03/30 Read block 492 len 28672
188742024-02-15T03:06:00.992Z04/30 Read block 451 len 32768
188752024-02-15T03:06:00.992Z05/30 Flush
188762024-02-15T03:06:00.993Z06/30 Write block 213 len 20480 data: 12 14 14 13 12
188772024-02-15T03:06:00.993Z07/30 Read block 155 len 24576
188782024-02-15T03:06:00.993Z08/30 Read block 546 len 40960
188792024-02-15T03:06:00.993Z09/30 Write block 561 len 40960 data: 14 13 12 13 16 12 12 11 10 12
188802024-02-15T03:06:00.993Z10/30 Read block 323 len 4096
188812024-02-15T03:06:00.993Z11/30 Flush
188822024-02-15T03:06:01.001Z12/30 Write block 580 len 36864 data: 16 12 14 13 12 13 13 11 9
188832024-02-15T03:06:01.001Z13/30 Write block 259 len 32768 data: 22 19 17 12 11 11 13 11
188842024-02-15T03:06:01.001Z14/30 Flush
188852024-02-15T03:06:01.001Z15/30 Write block 431 len 4096 data: 8
188862024-02-15T03:06:01.001Z16/30 Flush
188872024-02-15T03:06:01.001Z17/30 Write block 309 len 20480 data: 15 16 15 11 13
188882024-02-15T03:06:01.001Z18/30 Read block 51 len 4096
188892024-02-15T03:06:01.001Z19/30 Read block 100 len 20480
188902024-02-15T03:06:01.020Z20/30 Write block 500 len 28672 data: 15 14 15 16 16 14 14
188912024-02-15T03:06:01.020Z21/30 Read block 510 len 28672
188922024-02-15T03:06:01.021Z22/30 Read block 318 len 32768
188932024-02-15T03:06:01.021Z23/30 Write block 569 len 4096 data: 11
188942024-02-15T03:06:01.021Z24/30 Write block 74 len 16384 data: 9 12 12 9
188952024-02-15T03:06:01.021Z25/30 Write block 166 len 32768 data: 15 13 12 11 9 10 8 6
188962024-02-15T03:06:01.021Z26/30 Flush
188972024-02-15T03:06:01.021Z27/30 Write block 340 len 20480 data: 10 10 11 9 13
188982024-02-15T03:06:01.021Z28/30 Read block 279 len 20480
188992024-02-15T03:06:01.021Z29/30 Read block 284 len 16384
189002024-02-15T03:06:01.021Z30/30 Write block 545 len 28672 data: 9 11 11 12 10 9 8
189012024-02-15T03:06:01.021Z----------------------------------------------------------------
189022024-02-15T03:06:01.021Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
189032024-02-15T03:06:01.021ZStates: Active Active Active
189042024-02-15T03:06:01.021ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189052024-02-15T03:06:01.021Z 1 Acked 1000 Write 7 Done Done Done false
189062024-02-15T03:06:01.021Z 2 Acked 1001 Read 10 Done Done Done false
189072024-02-15T03:06:01.021Z 3 Acked 1002 Read 7 Done Done Done false
189082024-02-15T03:06:01.021Z 4 Acked 1003 Read 8 Done Done Done false
189092024-02-15T03:06:01.021Z 5 Acked 1004 Flush 0 Done Done Sent false
189102024-02-15T03:06:01.021Z 6 Acked 1005 Write 5 Done Done Sent false
189112024-02-15T03:06:01.021Z 7 Acked 1006 Read 6 Done Done Sent false
189122024-02-15T03:06:01.022Z 8 Acked 1007 Read 10 Done Done Sent false
189132024-02-15T03:06:01.022Z 9 Acked 1008 Write 10 Done Done Sent false
189142024-02-15T03:06:01.022Z 10 Acked 1009 Read 1 Done Done Sent false
189152024-02-15T03:06:01.022Z 11 Acked 1010 Flush 0 Done Done Sent false
189162024-02-15T03:06:01.022Z 12 Acked 1011 Write 9 Done Done Sent false
189172024-02-15T03:06:01.022Z 13 Acked 1012 Write 8 Done Done Sent false
189182024-02-15T03:06:01.022Z 14 Acked 1013 Flush 0 Done Done Sent false
189192024-02-15T03:06:01.022Z 15 Acked 1014 Write 1 Done Done Sent false
189202024-02-15T03:06:01.022Z 16 Acked 1015 Flush 0 Done Done Sent false
189212024-02-15T03:06:01.022Z 17 Acked 1016 Write 5 Done Done Sent false
189222024-02-15T03:06:01.022Z 18 Acked 1017 Read 1 Done Done Sent false
189232024-02-15T03:06:01.022Z 19 Acked 1018 Read 5 Done Done Sent false
189242024-02-15T03:06:01.022Z 20 Acked 1019 Write 7 Done Done Sent false
189252024-02-15T03:06:01.022Z 21 Acked 1020 Read 7 Done Done Sent false
189262024-02-15T03:06:01.022Z 22 Acked 1021 Read 8 Done Done Sent false
189272024-02-15T03:06:01.022Z 23 Acked 1022 Write 1 Done Done Sent false
189282024-02-15T03:06:01.022Z 24 Acked 1023 Write 4 Done Done Sent false
189292024-02-15T03:06:01.022Z 25 Acked 1024 Write 8 Done Done Sent false
189302024-02-15T03:06:01.022Z 26 Acked 1025 Flush 0 Done Done Sent false
189312024-02-15T03:06:01.022Z 27 Acked 1026 Write 5 Done Done Sent false
189322024-02-15T03:06:01.023Z 28 Acked 1027 Read 5 Done Done Sent false
189332024-02-15T03:06:01.023Z 29 Acked 1028 Read 4 Done Done Sent false
189342024-02-15T03:06:01.023Z 30 Acked 1029 Write 7 Sent Sent Sent false
189352024-02-15T03:06:01.023Z STATES DS:0 DS:1 DS:2 TOTAL
189362024-02-15T03:06:01.023Z New 0 0 0 0
189372024-02-15T03:06:01.023Z Sent 1 1 26 28
189382024-02-15T03:06:01.023Z Done 29 29 4 62
189392024-02-15T03:06:01.023Z Skipped 0 0 0 0
189402024-02-15T03:06:01.023Z Error 0 0 0 0
189412024-02-15T03:06:01.023ZLast Flush: 1025 1025 0
189422024-02-15T03:06:01.023ZDownstairs last five completed:
189432024-02-15T03:06:01.023ZUpstairs last five completed: 30 29 28 27 26
189442024-02-15T03:06:01.023Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:05:59.896088929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1824}
189452024-02-15T03:06:01.023ZWrote out file "/tmp/test_repair_verify.data"
189462024-02-15T03:06:01.023Z
189472024-02-15T03:06:01.033ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189482024-02-15T03:06:01.047ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189492024-02-15T03:06:01.047Z 0 000-019 168 168 168 276 276 276 F F F
189502024-02-15T03:06:01.047Z 1 020-039 169 169 169 277 277 277 F F F
189512024-02-15T03:06:01.047Z 2 040-059 166 166 166 274 274 274 F F F
189522024-02-15T03:06:01.047Z 3 060-079 170 170 168 282 282 276 F F F <---
189532024-02-15T03:06:01.047Z 4 080-099 165 165 165 271 271 271 F F F
189542024-02-15T03:06:01.047Z 5 100-119 165 165 165 271 271 271 F F F
189552024-02-15T03:06:01.047Z 6 120-139 167 167 167 275 275 275 F F F
189562024-02-15T03:06:01.048Z 7 140-159 170 170 160 278 278 262 F F T <---
189572024-02-15T03:06:01.048Z 8 160-179 170 170 169 282 282 277 F F T <---
189582024-02-15T03:06:01.048Z 9 180-199 169 169 169 277 277 277 F F F
189592024-02-15T03:06:01.048Z 10 200-219 170 170 164 279 279 268 F F F <---
189602024-02-15T03:06:01.048Z 11 220-239 168 168 168 276 276 276 F F F
189612024-02-15T03:06:01.048Z 12 240-259 170 170 167 280 280 275 F F F <---
189622024-02-15T03:06:01.049Z 13 260-279 170 170 167 280 280 275 F F F <---
189632024-02-15T03:06:01.049Z 14 280-299 165 165 165 271 271 271 F F F
189642024-02-15T03:06:01.049Z 15 300-319 170 170 166 282 282 273 F F F <---
189652024-02-15T03:06:01.049Z 16 320-339 161 161 161 263 263 263 F F F
189662024-02-15T03:06:01.049Z 17 340-359 159 159 159 260 260 260 T T F <---
189672024-02-15T03:06:01.049Z 18 360-379 169 169 169 277 277 277 F F F
189682024-02-15T03:06:01.049Z 19 380-399 166 166 166 274 274 274 F F F
189692024-02-15T03:06:01.050Z 20 400-419 168 168 168 276 276 276 F F F
189702024-02-15T03:06:01.050Z 21 420-439 170 170 168 281 281 276 F F F <---
189712024-02-15T03:06:01.050Z 22 440-459 165 165 165 271 271 271 F F F
189722024-02-15T03:06:01.050Z 23 460-479 166 166 166 272 272 272 F F F
189732024-02-15T03:06:01.050Z 24 480-499 166 166 166 274 274 274 F F F
189742024-02-15T03:06:01.050Z 25 500-519 170 170 169 282 282 277 F F F <---
189752024-02-15T03:06:01.050Z 26 520-539 168 168 168 276 276 276 F F F
189762024-02-15T03:06:01.050Z 27 540-559 159 159 159 260 260 260 T T F <---
189772024-02-15T03:06:01.050Z 28 560-579 170 170 164 282 282 270 F F F <---
189782024-02-15T03:06:01.051Z 29 580-599 170 170 168 280 280 276 F F F <---
189792024-02-15T03:06:01.051ZMax gen: 170, Max flush: 282
189802024-02-15T03:06:01.051ZError: Difference in extent metadata found!
189812024-02-15T03:06:01.051ZA Difference in extent metadata is expected here
189822024-02-15T03:06:01.051ZOn loop 85
189832024-02-15T03:06:01.051Z
189842024-02-15T03:06:01.057ZVerifying data now
189852024-02-15T03:06:06.713ZNo change after verify
189862024-02-15T03:06:06.713ZLoop: 85 Downstairs dump after verify (and repair):
189872024-02-15T03:06:06.743ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189882024-02-15T03:06:06.743Z 0 000-019 168 168 168 276 276 276 F F F
189892024-02-15T03:06:06.743Z 1 020-039 169 169 169 277 277 277 F F F
189902024-02-15T03:06:06.743Z 2 040-059 166 166 166 274 274 274 F F F
189912024-02-15T03:06:06.744Z 3 060-079 170 170 170 282 282 282 F F F
189922024-02-15T03:06:06.744Z 4 080-099 165 165 165 271 271 271 F F F
189932024-02-15T03:06:06.744Z 5 100-119 165 165 165 271 271 271 F F F
189942024-02-15T03:06:06.744Z 6 120-139 167 167 167 275 275 275 F F F
189952024-02-15T03:06:06.744Z 7 140-159 170 170 170 278 278 278 F F F
189962024-02-15T03:06:06.744Z 8 160-179 170 170 170 282 282 282 F F F
189972024-02-15T03:06:06.744Z 9 180-199 169 169 169 277 277 277 F F F
189982024-02-15T03:06:06.744Z 10 200-219 170 170 170 279 279 279 F F F
189992024-02-15T03:06:06.744Z 11 220-239 168 168 168 276 276 276 F F F
190002024-02-15T03:06:06.744Z 12 240-259 170 170 170 280 280 280 F F F
190012024-02-15T03:06:06.744Z 13 260-279 170 170 170 280 280 280 F F F
190022024-02-15T03:06:06.744Z 14 280-299 165 165 165 271 271 271 F F F
190032024-02-15T03:06:06.749Z 15 300-319 170 170 170 282 282 282 F F F
190042024-02-15T03:06:06.749Z 16 320-339 161 161 161 263 263 263 F F F
190052024-02-15T03:06:06.749Z 17 340-359 171 171 171 283 283 283 F F F
190062024-02-15T03:06:06.749Z 18 360-379 169 169 169 277 277 277 F F F
190072024-02-15T03:06:06.749Z 19 380-399 166 166 166 274 274 274 F F F
190082024-02-15T03:06:06.749Z 20 400-419 168 168 168 276 276 276 F F F
190092024-02-15T03:06:06.749Z 21 420-439 170 170 170 281 281 281 F F F
190102024-02-15T03:06:06.750Z 22 440-459 165 165 165 271 271 271 F F F
190112024-02-15T03:06:06.750Z 23 460-479 166 166 166 272 272 272 F F F
190122024-02-15T03:06:06.750Z 24 480-499 166 166 166 274 274 274 F F F
190132024-02-15T03:06:06.750Z 25 500-519 170 170 170 282 282 282 F F F
190142024-02-15T03:06:06.750Z 26 520-539 168 168 168 276 276 276 F F F
190152024-02-15T03:06:06.750Z 27 540-559 171 171 171 283 283 283 F F F
190162024-02-15T03:06:06.750Z 28 560-579 170 170 170 282 282 282 F F F
190172024-02-15T03:06:06.750Z 29 580-599 170 170 170 280 280 280 F F F
190182024-02-15T03:06:06.750ZMax gen: 171, Max flush: 283
190192024-02-15T03:06:06.750Z
190202024-02-15T03:06:06.750ZBegin loop 86
190212024-02-15T03:06:06.750ZDownstairs to restart: 0
190222024-02-15T03:06:06.756Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.63413071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832}
190232024-02-15T03:06:06.756Z{"msg":"Crucible Version: BuildInfo {\n version: Crucible runtime is spawned
190242024-02-15T03:06:06.756Z\"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634502523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832}
190252024-02-15T03:06:06.756Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634545298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832}
190262024-02-15T03:06:06.756Z{"msg":"Crucible f5746ec6-2017-426d-80c6-b14e017917c0 has session id: aabacd27-3cc4-430d-8e1c-9a6afa6cefbd","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634566005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190272024-02-15T03:06:06.757Z{"msg":"Upstairs opts: Upstairs UUID: f5746ec6-2017-426d-80c6-b14e017917c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.63460565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190282024-02-15T03:06:06.757Z{"msg":"Crucible stats registered with UUID: f5746ec6-2017-426d-80c6-b14e017917c0","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634633217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190292024-02-15T03:06:06.757Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634649515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832}
190302024-02-15T03:06:06.757Z{"msg":"f5746ec6-2017-426d-80c6-b14e017917c0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634664683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190312024-02-15T03:06:06.757Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634693169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190322024-02-15T03:06:06.757Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634733334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190332024-02-15T03:06:06.757Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.63477205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190342024-02-15T03:06:06.757Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634809135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190352024-02-15T03:06:06.757Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.63484844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190362024-02-15T03:06:06.757Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634875926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190372024-02-15T03:06:06.757Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634900643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190382024-02-15T03:06:06.757Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634940038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190392024-02-15T03:06:06.757Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.634968035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190402024-02-15T03:06:06.757Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.635030877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190412024-02-15T03:06:06.758Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.635058563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190422024-02-15T03:06:06.758Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:05.635294194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190432024-02-15T03:06:06.758Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:05.635350757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190442024-02-15T03:06:06.758Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:05.635418118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190452024-02-15T03:06:06.758Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.635449794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190462024-02-15T03:06:06.759Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.63548444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190472024-02-15T03:06:06.759Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.635511187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190482024-02-15T03:06:06.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.635755696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190492024-02-15T03:06:06.759Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.6358027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190502024-02-15T03:06:06.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.635826567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190512024-02-15T03:06:06.759Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.635864402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190522024-02-15T03:06:06.759Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.636300918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190532024-02-15T03:06:06.759Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.636420743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190542024-02-15T03:06:06.759Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.636818953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190552024-02-15T03:06:06.759Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.636859458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190562024-02-15T03:06:06.759Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.636874786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190572024-02-15T03:06:06.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.636958736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190582024-02-15T03:06:06.760Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.636985882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190592024-02-15T03:06:06.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:05.63700041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190602024-02-15T03:06:07.755Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.636676603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190612024-02-15T03:06:07.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.636764552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190622024-02-15T03:06:07.763Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.637674019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190632024-02-15T03:06:07.763Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.637717833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190642024-02-15T03:06:07.763Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.63821737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190652024-02-15T03:06:07.763Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.638802317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190662024-02-15T03:06:07.763Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.638836563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190672024-02-15T03:06:07.763Z{"msg":"[0]R flush_numbers[0..12]: [276, 277, 274, 282, 271, 271, 275, 278, 282, 277, 279, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.638855561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190682024-02-15T03:06:07.763Z{"msg":"[0]R generation[0..12]: [168, 169, 166, 170, 165, 165, 167, 170, 170, 169, 170, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.638879598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190692024-02-15T03:06:07.763Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.638966747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190702024-02-15T03:06:07.763Z{"msg":"[1]R flush_numbers[0..12]: [276, 277, 274, 282, 271, 271, 275, 278, 282, 277, 279, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639003042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190712024-02-15T03:06:07.764Z{"msg":"[1]R generation[0..12]: [168, 169, 166, 170, 165, 165, 167, 170, 170, 169, 170, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.63902017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190722024-02-15T03:06:07.764Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639040487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190732024-02-15T03:06:07.764Z{"msg":"[2]R flush_numbers[0..12]: [276, 277, 274, 282, 271, 271, 275, 278, 282, 277, 279, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639066024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190742024-02-15T03:06:07.764Z{"msg":"[2]R generation[0..12]: [168, 169, 166, 170, 165, 165, 167, 170, 170, 169, 170, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639081782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190752024-02-15T03:06:07.764ZWait for a query_work_queue command to finish before sending IO
190762024-02-15T03:06:07.764Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639119937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190772024-02-15T03:06:07.764Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639141145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190782024-02-15T03:06:07.764Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639156853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190792024-02-15T03:06:07.765Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639172841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190802024-02-15T03:06:07.765Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639204227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"":"downstairs","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190812024-02-15T03:06:07.765Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639222495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190822024-02-15T03:06:07.765Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639251841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"0","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190832024-02-15T03:06:07.765Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639291536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"1","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190842024-02-15T03:06:07.765Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639312893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"client":"2","":"downstairs","session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190852024-02-15T03:06:07.765Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.63934068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190862024-02-15T03:06:07.765Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639354908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190872024-02-15T03:06:07.766Z{"msg":"f5746ec6-2017-426d-80c6-b14e017917c0 is now active with session: aabacd27-3cc4-430d-8e1c-9a6afa6cefbd","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639370686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190882024-02-15T03:06:07.766Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639398733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832,"session_id":"aabacd27-3cc4-430d-8e1c-9a6afa6cefbd"}
190892024-02-15T03:06:07.766Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:06.639412841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832}
190902024-02-15T03:06:07.766ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
190912024-02-15T03:06:07.766ZLoading write count information from file "/tmp/test_repair_verify.data"
190922024-02-15T03:06:07.766ZRun Repair workload
190932024-02-15T03:06:07.767Z01/30 Write block 584 len 4096 data: 13
190942024-02-15T03:06:07.767Z02/30 Read block 549 len 12288
190952024-02-15T03:06:07.767Z03/30 Read block 468 len 12288
190962024-02-15T03:06:07.767Z04/30 Read block 516 len 24576
190972024-02-15T03:06:07.772Z05/30 Write block 403 len 36864 data: 10 9 14 14 16 16 22 24 24
190982024-02-15T03:06:07.772Z06/30 Read block 288 len 20480
190992024-02-15T03:06:07.772Z07/30 Read block 7 len 32768
191002024-02-15T03:06:07.772Z08/30 Read block 340 len 28672
191012024-02-15T03:06:07.772Z09/30 Read block 576 len 32768
191022024-02-15T03:06:07.773Z10/30 Write block 134 len 12288 data: 14 13 12
191032024-02-15T03:06:07.773Z11/30 Write block 187 len 32768 data: 15 16 19 16 15 15 20 20
191042024-02-15T03:06:07.773Z12/30 Flush
191052024-02-15T03:06:07.773Z13/30 Write block 254 len 12288 data: 15 13 15
191062024-02-15T03:06:07.773Z14/30 Read block 125 len 36864
191072024-02-15T03:06:07.782Z15/30 Write block 46 len 40960 data: 17 17 14 12 10 8 9 10 11 13
191082024-02-15T03:06:07.782Z16/30 Read block 275 len 40960
191092024-02-15T03:06:07.783Z17/30 Read block 443 len 8192
191102024-02-15T03:06:07.783Z18/30 Write block 0 len 32768 data: 3 3 4 8 11 11 12 11
191112024-02-15T03:06:07.783Z19/30 Read block 386 len 16384
191122024-02-15T03:06:07.783Z20/30 Read block 376 len 36864
191132024-02-15T03:06:07.783Z21/30 Read block 80 len 8192
191142024-02-15T03:06:07.783Z22/30 Write block 116 len 4096 data: 10
191152024-02-15T03:06:07.783Z23/30 Read block 484 len 4096
191162024-02-15T03:06:07.784Z24/30 Write block 505 len 16384 data: 15 15 11 11
191172024-02-15T03:06:07.784Z25/30 Read block 508 len 8192
191182024-02-15T03:06:07.784Z26/30 Read block 218 len 20480
191192024-02-15T03:06:07.790Z27/30 Write block 465 len 16384 data: 12 15 15 14
191202024-02-15T03:06:07.790Z28/30 Write block 45 len 32768 data: 21 18 18 15 13 11 9 10
191212024-02-15T03:06:07.790Z29/30 Write block 101 len 12288 data: 14 13 11
191222024-02-15T03:06:07.790Z30/30 Read block 163 len 4096
191232024-02-15T03:06:07.790Z----------------------------------------------------------------
191242024-02-15T03:06:07.790Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
191252024-02-15T03:06:07.790ZStates: Active Active Active
191262024-02-15T03:06:07.790ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191272024-02-15T03:06:07.791Z 1 Acked 1000 Write 1 Done Done Done false
191282024-02-15T03:06:07.791Z 2 Acked 1001 Read 3 Done Done Done false
191292024-02-15T03:06:07.791Z 3 Acked 1002 Read 3 Done Done Done false
191302024-02-15T03:06:07.791Z 4 Acked 1003 Read 6 Done Done Done false
191312024-02-15T03:06:07.791Z 5 Acked 1004 Write 9 Done Done Done false
191322024-02-15T03:06:07.791Z 6 Acked 1005 Read 5 Done Done Done false
191332024-02-15T03:06:07.791Z 7 Acked 1006 Read 8 Sent Done Done false
191342024-02-15T03:06:07.792Z 8 Acked 1007 Read 7 Sent Done Done false
191352024-02-15T03:06:07.792Z 9 Acked 1008 Read 8 Sent Done Done false
191362024-02-15T03:06:07.792Z 10 Acked 1009 Write 3 Sent Done Done false
191372024-02-15T03:06:07.792Z 11 Acked 1010 Write 8 Sent Done Done false
191382024-02-15T03:06:07.792Z 12 Acked 1011 Flush 0 Sent Done Done false
191392024-02-15T03:06:07.792Z 13 Acked 1012 Write 3 Sent Done Done false
191402024-02-15T03:06:07.792Z 14 Acked 1013 Read 9 Sent Done Done false
191412024-02-15T03:06:07.793Z 15 Acked 1014 Write 10 Sent Done Done false
191422024-02-15T03:06:07.793Z 16 Acked 1015 Read 10 Sent Done Done false
191432024-02-15T03:06:07.793Z 17 Acked 1016 Read 2 Sent Done Done false
191442024-02-15T03:06:07.794Z 18 Acked 1017 Write 8 Sent Done Done false
191452024-02-15T03:06:07.794Z 19 Acked 1018 Read 4 Sent Done Done false
191462024-02-15T03:06:07.794Z 20 Acked 1019 Read 9 Sent Done Done false
191472024-02-15T03:06:07.794Z 21 Acked 1020 Read 2 Sent Done Done false
191482024-02-15T03:06:07.794Z 22 Acked 1021 Write 1 Sent Done Done false
191492024-02-15T03:06:07.794Z 23 Acked 1022 Read 1 Sent Done Done false
191502024-02-15T03:06:07.794Z 24 Acked 1023 Write 4 Sent Done Done false
191512024-02-15T03:06:07.794Z 25 Acked 1024 Read 2 Sent Done Done false
191522024-02-15T03:06:07.794Z 26 Acked 1025 Read 5 Sent Done Done false
191532024-02-15T03:06:07.795Z 27 Acked 1026 Write 4 Sent Done Done false
191542024-02-15T03:06:07.795Z 28 Acked 1027 Write 8 Sent Done Done false
191552024-02-15T03:06:07.795Z 29 Acked 1028 Write 3 Sent Done Done false
191562024-02-15T03:06:07.795Z 30 Acked 1029 Read 1 Sent Sent Done false
191572024-02-15T03:06:07.795Z STATES DS:0 DS:1 DS:2 TOTAL
191582024-02-15T03:06:07.795Z New 0 0 0 0
191592024-02-15T03:06:07.795Z Sent 24 1 0 25
191602024-02-15T03:06:07.796Z Done 6 29 30 65
191612024-02-15T03:06:07.796Z Skipped 0 0 0 0
191622024-02-15T03:06:07.796Z Error 0 0 0 0
191632024-02-15T03:06:07.796ZLast Flush: 0 1011 1011
191642024-02-15T03:06:07.796ZDownstairs last five completed:
191652024-02-15T03:06:07.796ZUpstairs last five completed: 30 29 28 27 26
191662024-02-15T03:06:07.796Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:06.668852012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1832}
191672024-02-15T03:06:07.796ZWrote out file "/tmp/test_repair_verify.data"
191682024-02-15T03:06:07.865Z
191692024-02-15T03:06:07.865ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
191702024-02-15T03:06:07.865ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191712024-02-15T03:06:07.865Z 0 000-019 168 168 168 276 276 276 F T T <---
191722024-02-15T03:06:07.865Z 1 020-039 169 169 169 277 277 277 F F F
191732024-02-15T03:06:07.865Z 2 040-059 166 166 166 274 274 274 F T T <---
191742024-02-15T03:06:07.865Z 3 060-079 170 170 170 282 282 282 F F F
191752024-02-15T03:06:07.865Z 4 080-099 165 165 165 271 271 271 F F F
191762024-02-15T03:06:07.866Z 5 100-119 165 165 165 271 271 271 F T T <---
191772024-02-15T03:06:07.866Z 6 120-139 167 172 172 275 284 284 F F F <---
191782024-02-15T03:06:07.866Z 7 140-159 170 170 170 278 278 278 F F F
191792024-02-15T03:06:07.866Z 8 160-179 170 170 170 282 282 282 F F F
191802024-02-15T03:06:07.866Z 9 180-199 169 172 172 277 284 284 F F F <---
191812024-02-15T03:06:07.866Z 10 200-219 170 170 170 279 279 279 F F F
191822024-02-15T03:06:07.866Z 11 220-239 168 168 168 276 276 276 F F F
191832024-02-15T03:06:07.866Z 12 240-259 170 170 170 280 280 280 F T T <---
191842024-02-15T03:06:07.866Z 13 260-279 170 170 170 280 280 280 F F F
191852024-02-15T03:06:07.866Z 14 280-299 165 165 165 271 271 271 F F F
191862024-02-15T03:06:07.866Z 15 300-319 170 170 170 282 282 282 F F F
191872024-02-15T03:06:07.866Z 16 320-339 161 161 161 263 263 263 F F F
191882024-02-15T03:06:07.866Z 17 340-359 171 171 171 283 283 283 F F F
191892024-02-15T03:06:07.866Z 18 360-379 169 169 169 277 277 277 F F F
191902024-02-15T03:06:07.867Z 19 380-399 166 166 166 274 274 274 F F F
191912024-02-15T03:06:07.867Z 20 400-419 168 172 172 276 284 284 T F F <---
191922024-02-15T03:06:07.867Z 21 420-439 170 170 170 281 281 281 F F F
191932024-02-15T03:06:07.867Z 22 440-459 165 165 165 271 271 271 F F F
191942024-02-15T03:06:07.867Z 23 460-479 166 166 166 272 272 272 F T T <---
191952024-02-15T03:06:07.867Z 24 480-499 166 166 166 274 274 274 F F F
191962024-02-15T03:06:07.867Z 25 500-519 170 170 170 282 282 282 F T T <---
191972024-02-15T03:06:07.867Z 26 520-539 168 168 168 276 276 276 F F F
191982024-02-15T03:06:07.867Z 27 540-559 171 171 171 283 283 283 F F F
191992024-02-15T03:06:07.867Z 28 560-579 170 170 170 282 282 282 F F F
192002024-02-15T03:06:07.867Z 29 580-599 170 172 172 280 284 284 T F F <---
192012024-02-15T03:06:07.867ZMax gen: 172, Max flush: 284
192022024-02-15T03:06:07.868ZError: Difference in extent metadata found!
192032024-02-15T03:06:07.868ZA Difference in extent metadata is expected here
192042024-02-15T03:06:07.868ZOn loop 86
192052024-02-15T03:06:07.868Z
192062024-02-15T03:06:07.868ZVerifying data now
192072024-02-15T03:06:09.790ZNo change after verify
192082024-02-15T03:06:09.790ZLoop: 86 Downstairs dump after verify (and repair):
192092024-02-15T03:06:09.808ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192102024-02-15T03:06:09.808Z 0 000-019 173 173 173 285 285 285 F F F
192112024-02-15T03:06:09.808Z 1 020-039 169 169 169 277 277 277 F F F
192122024-02-15T03:06:09.808Z 2 040-059 173 173 173 285 285 285 F F F
192132024-02-15T03:06:09.808Z 3 060-079 170 170 170 282 282 282 F F F
192142024-02-15T03:06:09.809Z 4 080-099 165 165 165 271 271 271 F F F
192152024-02-15T03:06:09.809Z 5 100-119 173 173 173 285 285 285 F F F
192162024-02-15T03:06:09.809Z 6 120-139 172 172 172 284 284 284 F F F
192172024-02-15T03:06:09.809Z 7 140-159 170 170 170 278 278 278 F F F
192182024-02-15T03:06:09.809Z 8 160-179 170 170 170 282 282 282 F F F
192192024-02-15T03:06:09.809Z 9 180-199 172 172 172 284 284 284 F F F
192202024-02-15T03:06:09.810Z 10 200-219 170 170 170 279 279 279 F F F
192212024-02-15T03:06:09.810Z 11 220-239 168 168 168 276 276 276 F F F
192222024-02-15T03:06:09.810Z 12 240-259 173 173 173 285 285 285 F F F
192232024-02-15T03:06:09.810Z 13 260-279 170 170 170 280 280 280 F F F
192242024-02-15T03:06:09.810Z 14 280-299 165 165 165 271 271 271 F F F
192252024-02-15T03:06:09.810Z 15 300-319 170 170 170 282 282 282 F F F
192262024-02-15T03:06:09.810Z 16 320-339 161 161 161 263 263 263 F F F
192272024-02-15T03:06:09.810Z 17 340-359 171 171 171 283 283 283 F F F
192282024-02-15T03:06:09.811Z 18 360-379 169 169 169 277 277 277 F F F
192292024-02-15T03:06:09.811Z 19 380-399 166 166 166 274 274 274 F F F
192302024-02-15T03:06:09.811Z 20 400-419 172 172 172 284 284 284 F F F
192312024-02-15T03:06:09.811Z 21 420-439 170 170 170 281 281 281 F F F
192322024-02-15T03:06:09.812Z 22 440-459 165 165 165 271 271 271 F F F
192332024-02-15T03:06:09.812Z 23 460-479 173 173 173 285 285 285 F F F
192342024-02-15T03:06:09.812Z 24 480-499 166 166 166 274 274 274 F F F
192352024-02-15T03:06:09.812Z 25 500-519 173 173 173 285 285 285 F F F
192362024-02-15T03:06:09.812Z 26 520-539 168 168 168 276 276 276 F F F
192372024-02-15T03:06:09.812Z 27 540-559 171 171 171 283 283 283 F F F
192382024-02-15T03:06:09.812Z 28 560-579 170 170 170 282 282 282 F F F
192392024-02-15T03:06:09.813Z 29 580-599 172 172 172 284 284 284 F F F
192402024-02-15T03:06:09.813ZMax gen: 173, Max flush: 285
192412024-02-15T03:06:09.813Z
192422024-02-15T03:06:09.813ZBegin loop 87
192432024-02-15T03:06:09.813ZDownstairs to restart: 1
192442024-02-15T03:06:09.813Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274308903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840}
192452024-02-15T03:06:09.813Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274601117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840}
192462024-02-15T03:06:09.814Z{"msg":"Crucible runtime is spawned
192472024-02-15T03:06:09.814ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274647551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840}
192482024-02-15T03:06:09.814Z{"msg":"Crucible 173f3913-c58e-46f4-b501-ea5eb59b5390 has session id: 1dc83636-fa72-48db-b2fb-fb06ef07c5fe","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274687626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192492024-02-15T03:06:09.814Z{"msg":"Upstairs opts: Upstairs UUID: 173f3913-c58e-46f4-b501-ea5eb59b5390, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.27473547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192502024-02-15T03:06:09.814Z{"msg":"Crucible stats registered with UUID: 173f3913-c58e-46f4-b501-ea5eb59b5390","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274775065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192512024-02-15T03:06:09.814Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274799062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840}
192522024-02-15T03:06:09.815Z{"msg":"173f3913-c58e-46f4-b501-ea5eb59b5390 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274868333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192532024-02-15T03:06:09.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274906298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192542024-02-15T03:06:09.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.274935785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192552024-02-15T03:06:09.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.27496987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192562024-02-15T03:06:09.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275019164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192572024-02-15T03:06:09.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275060469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192582024-02-15T03:06:09.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275086866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192592024-02-15T03:06:09.815Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275125011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192602024-02-15T03:06:09.816Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275161917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192612024-02-15T03:06:09.816Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275183744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192622024-02-15T03:06:09.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275224049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192632024-02-15T03:06:09.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275248656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192642024-02-15T03:06:09.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.275283111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192652024-02-15T03:06:09.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.276026448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192662024-02-15T03:06:09.816Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.276068313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192672024-02-15T03:06:09.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.276095269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192682024-02-15T03:06:09.816Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.276126985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192692024-02-15T03:06:09.816Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.276584758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192702024-02-15T03:06:09.817Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.276756367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192712024-02-15T03:06:09.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277134509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192722024-02-15T03:06:09.817Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277180213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192732024-02-15T03:06:09.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.27721011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192742024-02-15T03:06:09.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277340773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192752024-02-15T03:06:09.817Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277393057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192762024-02-15T03:06:09.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277438671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192772024-02-15T03:06:09.817Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277477566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192782024-02-15T03:06:09.817Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277511772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192792024-02-15T03:06:09.817Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.277972384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192802024-02-15T03:06:09.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278441626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192812024-02-15T03:06:09.818Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.27848661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192822024-02-15T03:06:09.818Z{"msg":"[0]R flush_numbers[0..12]: [285, 277, 285, 282, 271, 285, 284, 278, 282, 284, 279, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278513526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192832024-02-15T03:06:09.818Z{"msg":"[0]R generation[0..12]: [173, 169, 173, 170, 165, 173, 172, 170, 170, 172, 170, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278549092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192842024-02-15T03:06:09.818Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278585737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192852024-02-15T03:06:09.818Z{"msg":"[1]R flush_numbers[0..12]: [285, 277, 285, 282, 271, 285, 284, 278, 282, 284, 279, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278621533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192862024-02-15T03:06:09.818Z{"msg":"[1]R generation[0..12]: [173, 169, 173, 170, 165, 173, 172, 170, 170, 172, 170, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278658099Z","hostname":Wait for a query_work_queue command to finish before sending IO
192872024-02-15T03:06:09.818Z"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192882024-02-15T03:06:09.818Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278700923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192892024-02-15T03:06:09.818Z{"msg":"[2]R flush_numbers[0..12]: [285, 277, 285, 282, 271, 285, 284, 278, 282, 284, 279, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278737838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192902024-02-15T03:06:09.819Z{"msg":"[2]R generation[0..12]: [173, 169, 173, 170, 165, 173, 172, 170, 170, 172, 170, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278776584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192912024-02-15T03:06:09.819Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278812319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192922024-02-15T03:06:09.819Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278859763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192932024-02-15T03:06:09.819Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278895539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192942024-02-15T03:06:09.819Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278933234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192952024-02-15T03:06:09.819Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.278969789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"":"downstairs","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192962024-02-15T03:06:09.819Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279004895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192972024-02-15T03:06:09.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279039751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"0","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192982024-02-15T03:06:09.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279068207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"1","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
192992024-02-15T03:06:09.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279104943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"client":"2","":"downstairs","session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
193002024-02-15T03:06:09.820Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279131089Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
193012024-02-15T03:06:09.820Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279167635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
193022024-02-15T03:06:09.820Z{"msg":"173f3913-c58e-46f4-b501-ea5eb59b5390 is now active with session: 1dc83636-fa72-48db-b2fb-fb06ef07c5fe","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279191621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
193032024-02-15T03:06:09.820Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279219518Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193042024-02-15T03:06:09.820Zhostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840,"session_id":"1dc83636-fa72-48db-b2fb-fb06ef07c5fe"}
193052024-02-15T03:06:09.820Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:08.279245425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840}
193062024-02-15T03:06:09.820ZLoading write count information from file "/tmp/test_repair_verify.data"
193072024-02-15T03:06:09.821ZRun Repair workload
193082024-02-15T03:06:09.821Z01/30 Write block 415 len 40960 data: 13 11 10 10 11 12 11 11 13 12
193092024-02-15T03:06:09.863Z02/30 Flush
193102024-02-15T03:06:09.863Z03/30 Write block 566 len 36864 data: 13 13 12 12 13 13 13 16 17
193112024-02-15T03:06:09.863Z04/30 Read block 236 len 16384
193122024-02-15T03:06:09.863Z05/30 Write block 184 len 40960 data: 16 17 16 16 17 20 17 16 16 21
193132024-02-15T03:06:09.863Z06/30 Write block 555 len 24576 data: 13 10 15 16 19 18
193142024-02-15T03:06:09.863Z07/30 Write block 555 len 16384 data: 14 11 16 17
193152024-02-15T03:06:09.864Z08/30 Write block 221 len 32768 data: 16 16 14 13 13 14 12 14
193162024-02-15T03:06:09.864Z09/30 Read block 212 len 4096
193172024-02-15T03:06:09.864Z10/30 Write block 182 len 20480 data: 11 14 17 18 17
193182024-02-15T03:06:09.864Z11/30 Write block 434 len 32768 data: 8 10 13 13 13 11 13 13
193192024-02-15T03:06:09.864Z12/30 Write block 579 len 40960 data: 14 17 13 15 14 14 14 14 12 10
193202024-02-15T03:06:09.864Z13/30 Read block 411 len 32768
193212024-02-15T03:06:09.864Z14/30 Write block 291 len 16384 data: 21 20 18 18
193222024-02-15T03:06:09.864Z15/30 Write block 413 len 4096 data: 22
193232024-02-15T03:06:09.864Z16/30 Write block 193 len 12288 data: 22 21 21
193242024-02-15T03:06:09.864Z17/30 Read block 340 len 4096
193252024-02-15T03:06:09.865Z18/30 Read block 430 len 16384
193262024-02-15T03:06:09.865Z19/30 Read block 85 len 12288
193272024-02-15T03:06:09.865Z20/30 Read block 277 len 8192
193282024-02-15T03:06:09.865Z21/30 Write block 281 len 24576 data: 10 8 11 13 14 15
193292024-02-15T03:06:09.865Z22/30 Read block 580 len 16384
193302024-02-15T03:06:09.865Z23/30 Write block 347 len 32768 data: 16 14 15 18 19 18 17 16
193312024-02-15T03:06:09.865Z24/30 Read block 582 len 8192
193322024-02-15T03:06:09.865Z25/30 Write block 15 len 8192 data: 8 8
193332024-02-15T03:06:09.866Z26/30 Write block 540 len 36864 data: 10 12 15 16 10 10 12 12 13
193342024-02-15T03:06:09.866Z27/30 Write block 40 len 24576 data: 14 15 16 17 19 22
193352024-02-15T03:06:09.866Z28/30 Write block 76 len 12288 data: 13 10 8
193362024-02-15T03:06:09.866Z29/30 Read block 533 len 12288
193372024-02-15T03:06:09.866Z30/30 Write block 460 len 20480 data: 14 13 13 10 10
193382024-02-15T03:06:09.866Z----------------------------------------------------------------
193392024-02-15T03:06:09.866Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
193402024-02-15T03:06:09.867ZStates: Active Active Active
193412024-02-15T03:06:09.867ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193422024-02-15T03:06:09.867Z 1 Acked 1000 Write 10 Done Sent Done false
193432024-02-15T03:06:09.867Z 2 Acked 1001 Flush 0 Done Sent Done false
193442024-02-15T03:06:09.867Z 3 Acked 1002 Write 9 Done Sent Done false
193452024-02-15T03:06:09.867Z 4 Acked 1003 Read 4 Done Sent Done false
193462024-02-15T03:06:09.867Z 5 Acked 1004 Write 10 Done Sent Done false
193472024-02-15T03:06:09.867Z 6 Acked 1005 Write 6 Done Sent Done false
193482024-02-15T03:06:09.867Z 7 Acked 1006 Write 4 Done Sent Done false
193492024-02-15T03:06:09.868Z 8 Acked 1007 Write 8 Done Sent Done false
193502024-02-15T03:06:09.868Z 9 Acked 1008 Read 1 Done Sent Done false
193512024-02-15T03:06:09.868Z 10 Acked 1009 Write 5 Done Sent Done false
193522024-02-15T03:06:09.868Z 11 Acked 1010 Write 8 Done Sent Done false
193532024-02-15T03:06:09.868Z 12 Acked 1011 Write 10 Done Sent Done false
193542024-02-15T03:06:09.868Z 13 Acked 1012 Read 8 Done Sent Done false
193552024-02-15T03:06:09.869Z 14 Acked 1013 Write 4 Done Sent Done false
193562024-02-15T03:06:09.869Z 15 Acked 1014 Write 1 Done Sent Done false
193572024-02-15T03:06:09.869Z 16 Acked 1015 Write 3 Done Sent Done false
193582024-02-15T03:06:09.869Z 17 Acked 1016 Read 1 Done Sent Done false
193592024-02-15T03:06:09.869Z 18 Acked 1017 Read 4 Done Sent Done false
193602024-02-15T03:06:09.869Z 19 Acked 1018 Read 3 Done Sent Done false
193612024-02-15T03:06:09.869Z 20 Acked 1019 Read 2 Done Sent Done false
193622024-02-15T03:06:09.869Z 21 Acked 1020 Write 6 Done Sent Done false
193632024-02-15T03:06:09.869Z 22 Acked 1021 Read 4 Done Sent Done false
193642024-02-15T03:06:09.869Z 23 Acked 1022 Write 8 Done Sent Done false
193652024-02-15T03:06:09.869Z 24 Acked 1023 Read 2 Done Sent Done false
193662024-02-15T03:06:09.869Z 25 Acked 1024 Write 2 Done Sent Done false
193672024-02-15T03:06:09.870Z 26 Acked 1025 Write 9 Done Sent Done false
193682024-02-15T03:06:09.870Z 27 Acked 1026 Write 6 Done Sent Done false
193692024-02-15T03:06:09.870Z 28 Acked 1027 Write 3 Done Sent Done false
193702024-02-15T03:06:09.870Z 29 Acked 1028 Read 3 Done Sent Done false
193712024-02-15T03:06:09.871Z 30 Acked 1029 Write 5 Sent Sent Sent false
193722024-02-15T03:06:09.871Z STATES DS:0 DS:1 DS:2 TOTAL
193732024-02-15T03:06:09.871Z New 0 0 0 0
193742024-02-15T03:06:09.872Z Sent 1 30 1 32
193752024-02-15T03:06:09.872Z Done 29 0 29 58
193762024-02-15T03:06:09.873Z Skipped 0 0 0 0
193772024-02-15T03:06:09.873Z Error 0 0 0 0
193782024-02-15T03:06:09.873ZLast Flush: 1001 0 1001
193792024-02-15T03:06:09.874ZDownstairs last five completed:
193802024-02-15T03:06:09.874ZUpstairs last five completed: 30 29 28 27 26
193812024-02-15T03:06:09.874Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:08.307305099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1840}
193822024-02-15T03:06:09.874ZWrote out file "/tmp/test_repair_verify.data"
193832024-02-15T03:06:09.874Z
193842024-02-15T03:06:09.874ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
193852024-02-15T03:06:09.875ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193862024-02-15T03:06:09.875Z 0 000-019 173 173 173 285 285 285 T F T <---
193872024-02-15T03:06:09.875Z 1 020-039 169 169 169 277 277 277 F F F
193882024-02-15T03:06:09.875Z 2 040-059 173 173 173 285 285 285 T F T <---
193892024-02-15T03:06:09.875Z 3 060-079 170 170 170 282 282 282 T F T <---
193902024-02-15T03:06:09.875Z 4 080-099 165 165 165 271 271 271 F F F
193912024-02-15T03:06:09.876Z 5 100-119 173 173 173 285 285 285 F F F
193922024-02-15T03:06:09.876Z 6 120-139 172 172 172 284 284 284 F F F
193932024-02-15T03:06:09.876Z 7 140-159 170 170 170 278 278 278 F F F
193942024-02-15T03:06:09.876Z 8 160-179 170 170 170 282 282 282 F F F
193952024-02-15T03:06:09.876Z 9 180-199 172 172 172 284 284 284 T F T <---
193962024-02-15T03:06:09.876Z 10 200-219 170 170 170 279 279 279 F F F
193972024-02-15T03:06:09.876Z 11 220-239 168 168 168 276 276 276 T F T <---
193982024-02-15T03:06:09.877Z 12 240-259 173 173 173 285 285 285 F F F
193992024-02-15T03:06:09.877Z 13 260-279 170 170 170 280 280 280 F F F
194002024-02-15T03:06:09.877Z 14 280-299 165 165 165 271 271 271 T F T <---
194012024-02-15T03:06:09.877Z 15 300-319 170 170 170 282 282 282 F F F
194022024-02-15T03:06:09.878Z 16 320-339 161 161 161 263 263 263 F F F
194032024-02-15T03:06:09.878Z 17 340-359 171 171 171 283 283 283 T F T <---
194042024-02-15T03:06:09.878Z 18 360-379 169 169 169 277 277 277 F F F
194052024-02-15T03:06:09.878Z 19 380-399 166 166 166 274 274 274 F F F
194062024-02-15T03:06:09.878Z 20 400-419 174 172 174 286 284 286 T F T <---
194072024-02-15T03:06:09.878Z 21 420-439 174 170 174 286 281 286 T F T <---
194082024-02-15T03:06:09.878Z 22 440-459 165 165 165 271 271 271 T F T <---
194092024-02-15T03:06:09.894Z 23 460-479 173 173 173 285 285 285 T F T <---
194102024-02-15T03:06:09.895Z 24 480-499 166 166 166 274 274 274 F F F
194112024-02-15T03:06:09.895Z 25 500-519 173 173 173 285 285 285 F F F
194122024-02-15T03:06:09.895Z 26 520-539 168 168 168 276 276 276 F F F
194132024-02-15T03:06:09.896Z 27 540-559 171 171 171 283 283 283 T F T <---
194142024-02-15T03:06:09.896Z 28 560-579 170 170 170 282 282 282 T F T <---
194152024-02-15T03:06:09.896Z 29 580-599 172 172 172 284 284 284 T F T <---
194162024-02-15T03:06:09.896ZMax gen: 174, Max flush: 286
194172024-02-15T03:06:09.896ZError: Difference in extent metadata found!
194182024-02-15T03:06:09.896ZA Difference in extent metadata is expected here
194192024-02-15T03:06:09.896ZOn loop 87
194202024-02-15T03:06:09.896Z
194212024-02-15T03:06:09.896ZVerifying data now
194222024-02-15T03:06:14.182ZNo change after verify
194232024-02-15T03:06:14.182ZLoop: 87 Downstairs dump after verify (and repair):
194242024-02-15T03:06:14.197ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194252024-02-15T03:06:14.197Z 0 000-019 175 175 175 287 287 287 F F F
194262024-02-15T03:06:14.198Z 1 020-039 169 169 169 277 277 277 F F F
194272024-02-15T03:06:14.198Z 2 040-059 175 175 175 287 287 287 F F F
194282024-02-15T03:06:14.198Z 3 060-079 175 175 175 287 287 287 F F F
194292024-02-15T03:06:14.198Z 4 080-099 165 165 165 271 271 271 F F F
194302024-02-15T03:06:14.198Z 5 100-119 173 173 173 285 285 285 F F F
194312024-02-15T03:06:14.198Z 6 120-139 172 172 172 284 284 284 F F F
194322024-02-15T03:06:14.198Z 7 140-159 170 170 170 278 278 278 F F F
194332024-02-15T03:06:14.199Z 8 160-179 170 170 170 282 282 282 F F F
194342024-02-15T03:06:14.199Z 9 180-199 175 175 175 287 287 287 F F F
194352024-02-15T03:06:14.199Z 10 200-219 170 170 170 279 279 279 F F F
194362024-02-15T03:06:14.199Z 11 220-239 175 175 175 287 287 287 F F F
194372024-02-15T03:06:14.199Z 12 240-259 173 173 173 285 285 285 F F F
194382024-02-15T03:06:14.211Z 13 260-279 170 170 170 280 280 280 F F F
194392024-02-15T03:06:14.211Z 14 280-299 175 175 175 287 287 287 F F F
194402024-02-15T03:06:14.211Z 15 300-319 170 170 170 282 282 282 F F F
194412024-02-15T03:06:14.211Z 16 320-339 161 161 161 263 263 263 F F F
194422024-02-15T03:06:14.211Z 17 340-359 175 175 175 287 287 287 F F F
194432024-02-15T03:06:14.212Z 18 360-379 169 169 169 277 277 277 F F F
194442024-02-15T03:06:14.212Z 19 380-399 166 166 166 274 274 274 F F F
194452024-02-15T03:06:14.212Z 20 400-419 175 175 175 287 287 287 F F F
194462024-02-15T03:06:14.212Z 21 420-439 175 175 175 287 287 287 F F F
194472024-02-15T03:06:14.212Z 22 440-459 175 175 175 287 287 287 F F F
194482024-02-15T03:06:14.212Z 23 460-479 175 175 175 287 287 287 F F F
194492024-02-15T03:06:14.212Z 24 480-499 166 166 166 274 274 274 F F F
194502024-02-15T03:06:14.213Z 25 500-519 173 173 173 285 285 285 F F F
194512024-02-15T03:06:14.213Z 26 520-539 168 168 168 276 276 276 F F F
194522024-02-15T03:06:14.213Z 27 540-559 175 175 175 287 287 287 F F F
194532024-02-15T03:06:14.213Z 28 560-579 175 175 175 287 287 287 F F F
194542024-02-15T03:06:14.213Z 29 580-599 175 175 175 287 287 287 F F F
194552024-02-15T03:06:14.213ZMax gen: 175, Max flush: 287
194562024-02-15T03:06:14.213Z
194572024-02-15T03:06:14.214ZBegin loop 88
194582024-02-15T03:06:14.214ZDownstairs to restart: 2
194592024-02-15T03:06:14.347Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.10326495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848}
194602024-02-15T03:06:14.347Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103555163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848}
194612024-02-15T03:06:14.356Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
194622024-02-15T03:06:14.356Z,"time":"2024-02-15T03:06:13.103595788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848}
194632024-02-15T03:06:14.356Z{"msg":"Crucible 3c6186e0-7e34-41c5-8668-6faf60ea7add has session id: cb556743-4af4-44b1-aeed-38ba7aa2ea31","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103631024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194642024-02-15T03:06:14.356Z{"msg":"Upstairs opts: Upstairs UUID: 3c6186e0-7e34-41c5-8668-6faf60ea7add, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103668299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194652024-02-15T03:06:14.356Z{"msg":"Crucible stats registered with UUID: 3c6186e0-7e34-41c5-8668-6faf60ea7add","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103693006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194662024-02-15T03:06:14.357Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103727042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848}
194672024-02-15T03:06:14.357Z{"msg":"3c6186e0-7e34-41c5-8668-6faf60ea7add active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103864505Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194682024-02-15T03:06:14.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103892671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194692024-02-15T03:06:14.357Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103908469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194702024-02-15T03:06:14.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103923377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194712024-02-15T03:06:14.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.103964862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194722024-02-15T03:06:14.358Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104003717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194732024-02-15T03:06:14.358Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104043502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194742024-02-15T03:06:14.358Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104070629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194752024-02-15T03:06:14.358Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104100325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194762024-02-15T03:06:14.358Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104128662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194772024-02-15T03:06:14.358Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:13.104161467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194782024-02-15T03:06:14.358Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:13.104199523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194792024-02-15T03:06:14.358Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:13.104226689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194802024-02-15T03:06:14.359Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104261135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194812024-02-15T03:06:14.359Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104295131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194822024-02-15T03:06:14.359Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104319108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194832024-02-15T03:06:14.359Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104352863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194842024-02-15T03:06:14.359Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104395738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194852024-02-15T03:06:14.359Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104910064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194862024-02-15T03:06:14.360Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104946679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194872024-02-15T03:06:14.360Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104962207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194882024-02-15T03:06:14.360Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.104976025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194892024-02-15T03:06:14.360Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.105583849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194902024-02-15T03:06:14.360Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.105717683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194912024-02-15T03:06:14.360Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.106152768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194922024-02-15T03:06:14.360Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.106228988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194932024-02-15T03:06:14.361Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.106265324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194942024-02-15T03:06:14.361Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.106288681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194952024-02-15T03:06:14.361Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.106317067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194962024-02-15T03:06:14.361Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:13.106341354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194972024-02-15T03:06:15.224Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.105983442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194982024-02-15T03:06:15.224Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.106122375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
194992024-02-15T03:06:15.229Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.106970439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195002024-02-15T03:06:15.229Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.107006884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195012024-02-15T03:06:15.229Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.107537348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195022024-02-15T03:06:15.230Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108123184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195032024-02-15T03:06:15.230Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108161209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195042024-02-15T03:06:15.230Z{"msg":"[0]R flush_numbers[0..12]: [287, 277, 287, 287, 271, 285, 284, 278, 282, 287, 279, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108177767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195052024-02-15T03:06:15.230Z{"msg":"[0]R generation[0..12]: [175, 169, 175, 175, 165, 173, 172, 170, 170, 175, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108192196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195062024-02-15T03:06:15.230Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108215543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195072024-02-15T03:06:15.230Z{"msg":"[1]R flush_numbers[0..12]: [287, 277, 287, 287, 271, 285, 284, 278, 282, 287, 279, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108230631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195082024-02-15T03:06:15.230Z{"msg":"[1]R generation[0..12]: [175, 169, 175, 175, 165, 173, 172, 170, 170, 175, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108245669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195092024-02-15T03:06:15.230Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108260557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195102024-02-15T03:06:15.230Z{"msg":"[2]R flush_numbers[0..12]: [287, 277, 287, 287, 271, 285, 284, 278, 282, 287, 279, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108274795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195112024-02-15T03:06:15.230Z{"msg":"[2]R generation[0..12]: [175, 169, 175, 175, 165, 173, 172, 170, 170, 175, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108296592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195122024-02-15T03:06:15.230Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108311051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195132024-02-15T03:06:15.230Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108325859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195142024-02-15T03:06:15.230Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108339617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195152024-02-15T03:06:15.231Z{"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108359664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195162024-02-15T03:06:15.231ZWait for a query_work_queue command to finish before sending IO
195172024-02-15T03:06:15.231Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108382682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"":"downstairs","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195182024-02-15T03:06:15.231Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108402769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195192024-02-15T03:06:15.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108415288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"0","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195202024-02-15T03:06:15.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108428756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"1","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195212024-02-15T03:06:15.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108448553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"client":"2","":"downstairs","session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195222024-02-15T03:06:15.231Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108462042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195232024-02-15T03:06:15.231Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.10847482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195242024-02-15T03:06:15.231Z{"msg":"3c6186e0-7e34-41c5-8668-6faf60ea7add is now active with session: cb556743-4af4-44b1-aeed-38ba7aa2ea31","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108495358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195252024-02-15T03:06:15.231Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108507596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848,"session_id":"cb556743-4af4-44b1-aeed-38ba7aa2ea31"}
195262024-02-15T03:06:15.231Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:14.108520774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848}
195272024-02-15T03:06:15.231ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195282024-02-15T03:06:15.231ZLoading write count information from file "/tmp/test_repair_verify.data"
195292024-02-15T03:06:15.232ZRun Repair workload
195302024-02-15T03:06:15.232Z01/30 Write block 411 len 40960 data: 25 22 23 17 14 12 11 11 12 13
195312024-02-15T03:06:15.236Z02/30 Read block 558 len 4096
195322024-02-15T03:06:15.237Z03/30 Read block 104 len 36864
195332024-02-15T03:06:15.237Z04/30 Flush
195342024-02-15T03:06:15.237Z05/30 Write block 295 len 24576 data: 16 15 13 13 13 12
195352024-02-15T03:06:15.237Z06/30 Read block 62 len 16384
195362024-02-15T03:06:15.237Z07/30 Flush
195372024-02-15T03:06:15.266Z08/30 Write block 433 len 40960 data: 9 9 11 14 14 14 12 14 14 15
195382024-02-15T03:06:15.266Z09/30 Write block 208 len 8192 data: 12 12
195392024-02-15T03:06:15.266Z10/30 Read block 474 len 12288
195402024-02-15T03:06:15.266Z11/30 Read block 208 len 32768
195412024-02-15T03:06:15.266Z12/30 Read block 229 len 36864
195422024-02-15T03:06:15.266Z13/30 Read block 236 len 20480
195432024-02-15T03:06:15.266Z14/30 Read block 402 len 8192
195442024-02-15T03:06:15.266Z15/30 Write block 570 len 16384 data: 14 14 14 17
195452024-02-15T03:06:15.267Z16/30 Read block 132 len 36864
195462024-02-15T03:06:15.267Z17/30 Read block 10 len 12288
195472024-02-15T03:06:15.267Z18/30 Read block 193 len 36864
195482024-02-15T03:06:15.272Z19/30 Write block 95 len 8192 data: 11 15
195492024-02-15T03:06:15.272Z20/30 Read block 184 len 24576
195502024-02-15T03:06:15.272Z21/30 Write block 432 len 28672 data: 9 10 10 12 15 15 15
195512024-02-15T03:06:15.272Z22/30 Write block 432 len 12288 data: 10 11 11
195522024-02-15T03:06:15.272Z23/30 Write block 279 len 36864 data: 12 12 11 9 12 14 15 16 17
195532024-02-15T03:06:15.272Z24/30 Read block 134 len 20480
195542024-02-15T03:06:15.272Z25/30 Write block 223 len 16384 data: 15 14 14 15
195552024-02-15T03:06:15.273Z26/30 Write block 139 len 20480 data: 12 14 16 14 13
195562024-02-15T03:06:15.273Z27/30 Write block 61 len 4096 data: 13
195572024-02-15T03:06:15.273Z28/30 Write block 537 len 28672 data: 14 11 11 11 13 16 17
195582024-02-15T03:06:15.273Z29/30 Write block 83 len 40960 data: 7 11 10 13 13 10 9 10 11 9
195592024-02-15T03:06:15.273Z30/30 Read block 88 len 40960
195602024-02-15T03:06:15.273Z----------------------------------------------------------------
195612024-02-15T03:06:15.273Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:26
195622024-02-15T03:06:15.273ZStates: Active Active Active
195632024-02-15T03:06:15.273ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195642024-02-15T03:06:15.273Z 5 Acked 1004 Write 6 Done Done Sent false
195652024-02-15T03:06:15.273Z 6 Acked 1005 Read 4 Done Done Sent false
195662024-02-15T03:06:15.273Z 7 Acked 1006 Flush 0 Done Done Sent false
195672024-02-15T03:06:15.273Z 8 Acked 1007 Write 10 Done Done Sent false
195682024-02-15T03:06:15.274Z 9 Acked 1008 Write 2 Done Done Sent false
195692024-02-15T03:06:15.274Z 10 Acked 1009 Read 3 Done Done Sent false
195702024-02-15T03:06:15.274Z 11 Acked 1010 Read 8 Done Done Sent false
195712024-02-15T03:06:15.274Z 12 Acked 1011 Read 9 Done Done Sent false
195722024-02-15T03:06:15.274Z 13 Acked 1012 Read 5 Done Done Sent false
195732024-02-15T03:06:15.274Z 14 Acked 1013 Read 2 Done Done Sent false
195742024-02-15T03:06:15.274Z 15 Acked 1014 Write 4 Done Done Sent false
195752024-02-15T03:06:15.274Z 16 Acked 1015 Read 9 Done Done Sent false
195762024-02-15T03:06:15.274Z 17 Acked 1016 Read 3 Done Done Sent false
195772024-02-15T03:06:15.274Z 18 Acked 1017 Read 9 Done Done Sent false
195782024-02-15T03:06:15.275Z 19 Acked 1018 Write 2 Done Done Sent false
195792024-02-15T03:06:15.275Z 20 Acked 1019 Read 6 Done Done Sent false
195802024-02-15T03:06:15.275Z 21 Acked 1020 Write 7 Done Done Sent false
195812024-02-15T03:06:15.275Z 22 Acked 1021 Write 3 Done Done Sent false
195822024-02-15T03:06:15.275Z 23 Acked 1022 Write 9 Done Done Sent false
195832024-02-15T03:06:15.275Z 24 Acked 1023 Read 5 Done Done Sent false
195842024-02-15T03:06:15.275Z 25 Acked 1024 Write 4 Done Done Sent false
195852024-02-15T03:06:15.275Z 26 Acked 1025 Write 5 Done Done Sent false
195862024-02-15T03:06:15.275Z 27 Acked 1026 Write 1 Done Done Sent false
195872024-02-15T03:06:15.275Z 28 Acked 1027 Write 7 Done Done Sent false
195882024-02-15T03:06:15.275Z 29 Acked 1028 Write 10 Done Done Sent false
195892024-02-15T03:06:15.275Z 30 Acked 1029 Read 10 Done Done Sent false
195902024-02-15T03:06:15.276Z STATES DS:0 DS:1 DS:2 TOTAL
195912024-02-15T03:06:15.276Z New 0 0 0 0
195922024-02-15T03:06:15.276Z Sent 0 0 26 26
195932024-02-15T03:06:15.276Z Done 26 26 0 52
195942024-02-15T03:06:15.276Z Skipped 0 0 0 0
195952024-02-15T03:06:15.276Z Error 0 0 0 0
195962024-02-15T03:06:15.276ZLast Flush: 1006 1006 1003
195972024-02-15T03:06:15.276ZDownstairs last five completed: 1003 1002 1001 1000
195982024-02-15T03:06:15.277ZUpstairs last five completed: 30 29 28 27 26
195992024-02-15T03:06:15.277Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:14.140932223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1848}
196002024-02-15T03:06:15.277ZWrote out file "/tmp/test_repair_verify.data"
196012024-02-15T03:06:15.277Z
196022024-02-15T03:06:15.277ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196032024-02-15T03:06:15.293ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196042024-02-15T03:06:15.293Z 0 000-019 175 175 175 287 287 287 F F F
196052024-02-15T03:06:15.293Z 1 020-039 169 169 169 277 277 277 F F F
196062024-02-15T03:06:15.294Z 2 040-059 175 175 175 287 287 287 F F F
196072024-02-15T03:06:15.294Z 3 060-079 175 175 175 287 287 287 T T F <---
196082024-02-15T03:06:15.294Z 4 080-099 165 165 165 271 271 271 T T F <---
196092024-02-15T03:06:15.294Z 5 100-119 173 173 173 285 285 285 F F F
196102024-02-15T03:06:15.294Z 6 120-139 172 172 172 284 284 284 T T F <---
196112024-02-15T03:06:15.294Z 7 140-159 170 170 170 278 278 278 T T F <---
196122024-02-15T03:06:15.294Z 8 160-179 170 170 170 282 282 282 F F F
196132024-02-15T03:06:15.294Z 9 180-199 175 175 175 287 287 287 F F F
196142024-02-15T03:06:15.294Z 10 200-219 170 170 170 279 279 279 T T F <---
196152024-02-15T03:06:15.294Z 11 220-239 175 175 175 287 287 287 T T F <---
196162024-02-15T03:06:15.294Z 12 240-259 173 173 173 285 285 285 F F F
196172024-02-15T03:06:15.294Z 13 260-279 170 170 170 280 280 280 T T F <---
196182024-02-15T03:06:15.294Z 14 280-299 176 176 175 289 289 287 T T F <---
196192024-02-15T03:06:15.295Z 15 300-319 176 176 170 289 289 282 F F F <---
196202024-02-15T03:06:15.298Z 16 320-339 161 161 161 263 263 263 F F F
196212024-02-15T03:06:15.298Z 17 340-359 175 175 175 287 287 287 F F F
196222024-02-15T03:06:15.299Z 18 360-379 169 169 169 277 277 277 F F F
196232024-02-15T03:06:15.299Z 19 380-399 166 166 166 274 274 274 F F F
196242024-02-15T03:06:15.299Z 20 400-419 176 176 176 288 288 288 F F F
196252024-02-15T03:06:15.299Z 21 420-439 176 176 176 288 288 288 T T F <---
196262024-02-15T03:06:15.299Z 22 440-459 175 175 175 287 287 287 T T F <---
196272024-02-15T03:06:15.299Z 23 460-479 175 175 175 287 287 287 F F F
196282024-02-15T03:06:15.299Z 24 480-499 166 166 166 274 274 274 F F F
196292024-02-15T03:06:15.299Z 25 500-519 173 173 173 285 285 285 F F F
196302024-02-15T03:06:15.299Z 26 520-539 168 168 168 276 276 276 T T F <---
196312024-02-15T03:06:15.299Z 27 540-559 175 175 175 287 287 287 T T F <---
196322024-02-15T03:06:15.299Z 28 560-579 175 175 175 287 287 287 T T F <---
196332024-02-15T03:06:15.299Z 29 580-599 175 175 175 287 287 287 F F F
196342024-02-15T03:06:15.299ZMax gen: 176, Max flush: 289
196352024-02-15T03:06:15.299ZError: Difference in extent metadata found!
196362024-02-15T03:06:15.300ZA Difference in extent metadata is expected here
196372024-02-15T03:06:15.300ZOn loop 88
196382024-02-15T03:06:15.300Z
196392024-02-15T03:06:15.300ZVerifying data now
196402024-02-15T03:06:20.018ZNo change after verify
196412024-02-15T03:06:20.018ZLoop: 88 Downstairs dump after verify (and repair):
196422024-02-15T03:06:20.032ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196432024-02-15T03:06:20.033Z 0 000-019 175 175 175 287 287 287 F F F
196442024-02-15T03:06:20.033Z 1 020-039 169 169 169 277 277 277 F F F
196452024-02-15T03:06:20.033Z 2 040-059 175 175 175 287 287 287 F F F
196462024-02-15T03:06:20.033Z 3 060-079 177 177 177 290 290 290 F F F
196472024-02-15T03:06:20.033Z 4 080-099 177 177 177 290 290 290 F F F
196482024-02-15T03:06:20.034Z 5 100-119 173 173 173 285 285 285 F F F
196492024-02-15T03:06:20.034Z 6 120-139 177 177 177 290 290 290 F F F
196502024-02-15T03:06:20.034Z 7 140-159 177 177 177 290 290 290 F F F
196512024-02-15T03:06:20.034Z 8 160-179 170 170 170 282 282 282 F F F
196522024-02-15T03:06:20.034Z 9 180-199 175 175 175 287 287 287 F F F
196532024-02-15T03:06:20.034Z 10 200-219 177 177 177 290 290 290 F F F
196542024-02-15T03:06:20.034Z 11 220-239 177 177 177 290 290 290 F F F
196552024-02-15T03:06:20.035Z 12 240-259 173 173 173 285 285 285 F F F
196562024-02-15T03:06:20.035Z 13 260-279 177 177 177 290 290 290 F F F
196572024-02-15T03:06:20.035Z 14 280-299 177 177 177 290 290 290 F F F
196582024-02-15T03:06:20.035Z 15 300-319 176 176 176 289 289 289 F F F
196592024-02-15T03:06:20.035Z 16 320-339 161 161 161 263 263 263 F F F
196602024-02-15T03:06:20.035Z 17 340-359 175 175 175 287 287 287 F F F
196612024-02-15T03:06:20.036Z 18 360-379 169 169 169 277 277 277 F F F
196622024-02-15T03:06:20.036Z 19 380-399 166 166 166 274 274 274 F F F
196632024-02-15T03:06:20.036Z 20 400-419 176 176 176 288 288 288 F F F
196642024-02-15T03:06:20.036Z 21 420-439 177 177 177 290 290 290 F F F
196652024-02-15T03:06:20.036Z 22 440-459 177 177 177 290 290 290 F F F
196662024-02-15T03:06:20.037Z 23 460-479 175 175 175 287 287 287 F F F
196672024-02-15T03:06:20.037Z 24 480-499 166 166 166 274 274 274 F F F
196682024-02-15T03:06:20.037Z 25 500-519 173 173 173 285 285 285 F F F
196692024-02-15T03:06:20.037Z 26 520-539 177 177 177 290 290 290 F F F
196702024-02-15T03:06:20.037Z 27 540-559 177 177 177 290 290 290 F F F
196712024-02-15T03:06:20.037Z 28 560-579 177 177 177 290 290 290 F F F
196722024-02-15T03:06:20.037Z 29 580-599 175 175 175 287 287 287 F F F
196732024-02-15T03:06:20.037ZMax gen: 177, Max flush: 290
196742024-02-15T03:06:20.050Z
196752024-02-15T03:06:20.051ZBegin loop 89
196762024-02-15T03:06:20.051ZDownstairs to restart: 2
196772024-02-15T03:06:20.074Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.935922575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856}
196782024-02-15T03:06:20.074Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
196792024-02-15T03:06:20.075Z,"time":"2024-02-15T03:06:18.936255983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856}
196802024-02-15T03:06:20.075Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936306057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856}
196812024-02-15T03:06:20.075Z{"msg":"Crucible 8ee5e884-e0f8-4a3e-952b-6ab14ca6a480 has session id: ebc441de-e1ef-4b97-81c1-340cd9574bf8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936331394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196822024-02-15T03:06:20.075Z{"msg":"Upstairs opts: Upstairs UUID: 8ee5e884-e0f8-4a3e-952b-6ab14ca6a480, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936381387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196832024-02-15T03:06:20.076Z{"msg":"Crucible stats registered with UUID: 8ee5e884-e0f8-4a3e-952b-6ab14ca6a480","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936409114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196842024-02-15T03:06:20.076Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.93644475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856}
196852024-02-15T03:06:20.076Z{"msg":"8ee5e884-e0f8-4a3e-952b-6ab14ca6a480 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936527759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196862024-02-15T03:06:20.076Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936590261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196872024-02-15T03:06:20.076Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936650434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196882024-02-15T03:06:20.076Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936685899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196892024-02-15T03:06:20.076Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936721615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196902024-02-15T03:06:20.077Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936748252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196912024-02-15T03:06:20.077Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936773348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196922024-02-15T03:06:20.077Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936810394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196932024-02-15T03:06:20.077Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.93684002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196942024-02-15T03:06:20.077Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936876416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196952024-02-15T03:06:20.078Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.936904932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196962024-02-15T03:06:20.078Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:18.936951886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196972024-02-15T03:06:20.078Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.937040795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196982024-02-15T03:06:20.078Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:18.937061692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
196992024-02-15T03:06:20.078Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:18.937101097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197002024-02-15T03:06:20.078Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.937123514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197012024-02-15T03:06:20.079Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.937137453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197022024-02-15T03:06:20.079Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.937275286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197032024-02-15T03:06:20.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.937776543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197042024-02-15T03:06:20.114Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.937812618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197052024-02-15T03:06:20.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.937838115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197062024-02-15T03:06:20.114Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.93787422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197072024-02-15T03:06:20.114Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.938422822Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197082024-02-15T03:06:20.114Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.938459087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197092024-02-15T03:06:20.114Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.939000219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197102024-02-15T03:06:20.114Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.939037025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197112024-02-15T03:06:20.115Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.939052533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197122024-02-15T03:06:20.115Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.93907942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197132024-02-15T03:06:20.115Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.939101797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197142024-02-15T03:06:20.115Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:18.939115995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197152024-02-15T03:06:21.062Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.937716434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197162024-02-15T03:06:21.062Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.937809002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197172024-02-15T03:06:21.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.938679083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197182024-02-15T03:06:21.605Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.938718198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197192024-02-15T03:06:21.605Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939207277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197202024-02-15T03:06:21.605Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939783904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197212024-02-15T03:06:21.606Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.93981771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197222024-02-15T03:06:21.606Z{"msg":"[0]R flush_numbers[0..12]: [287, 277, 287, 290, 290, 285, 290, 290, 282, 287, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939836958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197232024-02-15T03:06:21.606Z{"msg":"[0]R generation[0..12]: [175, 169, 175, 177, 177, 173, 177, 177, 170, 175, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939857635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197242024-02-15T03:06:21.606Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939893111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197252024-02-15T03:06:21.606Z{"msg":"[1]R flush_numbers[0..12]: [287, 277, 287, 290, 290, 285, 290, 290, 282, 287, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939912538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197262024-02-15T03:06:21.606Z{"msg":"[1]R generation[0..12]: [175, 169, 175, 177, 177, 173, 177, 177, 170, 175, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939938625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197272024-02-15T03:06:21.606Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939956583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197282024-02-15T03:06:21.607Z{"msg":"[2]R flush_numbers[0..12]: [287, 277, 287, 290, 290, 285, 290, 290, 282, 287, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.939982779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197292024-02-15T03:06:21.607Z{"msg":"[2]R generation[0..12]: [175, 169, 175, 177, 177, 173, 177, 177, 170, 175, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940001847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197302024-02-15T03:06:21.607Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940029234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856Wait for a query_work_queue command to finish before sending IO
197312024-02-15T03:06:21.607Z,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197322024-02-15T03:06:21.607Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.94006192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197332024-02-15T03:06:21.607Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940092066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197342024-02-15T03:06:21.607Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940109874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197352024-02-15T03:06:21.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940133091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"":"downstairs","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197362024-02-15T03:06:21.607Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940146709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197372024-02-15T03:06:21.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940165496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"0","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197382024-02-15T03:06:21.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940179055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"1","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197392024-02-15T03:06:21.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940198822Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"client":"2","":"downstairs","session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197402024-02-15T03:06:21.607Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940212411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197412024-02-15T03:06:21.607Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940231608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197422024-02-15T03:06:21.608Z{"msg":"8ee5e884-e0f8-4a3e-952b-6ab14ca6a480 is now active with session: ebc441de-e1ef-4b97-81c1-340cd9574bf8","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940244887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197432024-02-15T03:06:21.608Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940263454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856,"session_id":"ebc441de-e1ef-4b97-81c1-340cd9574bf8"}
197442024-02-15T03:06:21.608Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:19.940275953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856}
197452024-02-15T03:06:21.608ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197462024-02-15T03:06:21.608ZLoading write count information from file "/tmp/test_repair_verify.data"
197472024-02-15T03:06:21.608ZRun Repair workload
197482024-02-15T03:06:21.608Z01/30 Flush
197492024-02-15T03:06:21.608Z02/30 Write block 373 len 20480 data: 13 11 11 12 14
197502024-02-15T03:06:21.608Z03/30 Write block 203 len 28672 data: 10 11 12 10 11 13 13
197512024-02-15T03:06:21.608Z04/30 Flush
197522024-02-15T03:06:21.608Z05/30 Write block 461 len 4096 data: 14
197532024-02-15T03:06:21.608Z06/30 Read block 207 len 20480
197542024-02-15T03:06:21.608Z07/30 Write block 276 len 36864 data: 14 15 11 13 13 12 10 13 15
197552024-02-15T03:06:21.608Z08/30 Flush
197562024-02-15T03:06:21.608Z09/30 Write block 293 len 8192 data: 19 19
197572024-02-15T03:06:21.649Z10/30 Read block 78 len 40960
197582024-02-15T03:06:21.649Z11/30 Read block 149 len 20480
197592024-02-15T03:06:21.649Z12/30 Read block 357 len 8192
197602024-02-15T03:06:21.649Z13/30 Write block 217 len 16384 data: 13 12 14 15
197612024-02-15T03:06:21.649Z14/30 Write block 76 len 32768 data: 14 11 9 8 7 8 8 8
197622024-02-15T03:06:21.650Z15/30 Write block 325 len 28672 data: 14 15 13 12 14 15 17
197632024-02-15T03:06:21.650Z16/30 Write block 525 len 28672 data: 7 6 10 12 15 15 11
197642024-02-15T03:06:21.650Z17/30 Flush
197652024-02-15T03:06:21.650Z18/30 Write block 15 len 8192 data: 9 9
197662024-02-15T03:06:21.650Z19/30 Read block 262 len 24576
197672024-02-15T03:06:21.650Z20/30 Flush
197682024-02-15T03:06:21.651Z21/30 Flush
197692024-02-15T03:06:21.651Z22/30 Write block 64 len 8192 data: 17 15
197702024-02-15T03:06:21.651Z23/30 Read block 192 len 4096
197712024-02-15T03:06:21.651Z24/30 Read block 382 len 4096
197722024-02-15T03:06:21.651Z25/30 Write block 153 len 8192 data: 13 13
197732024-02-15T03:06:21.651Z26/30 Write block 404 len 36864 data: 10 15 15 17 17 23 25 26 23
197742024-02-15T03:06:21.652Z27/30 Read block 261 len 36864
197752024-02-15T03:06:21.652Z28/30 Read block 309 len 36864
197762024-02-15T03:06:21.652Z29/30 Read block 244 len 12288
197772024-02-15T03:06:21.652Z30/30 Write block 77 len 32768 data: 12 10 9 8 9 9 9 12
197782024-02-15T03:06:21.653Z----------------------------------------------------------------
197792024-02-15T03:06:21.653Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
197802024-02-15T03:06:21.653ZStates: Active Active Active
197812024-02-15T03:06:21.653ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
197822024-02-15T03:06:21.653Z 1 Acked 1000 Flush 0 Done Done Sent false
197832024-02-15T03:06:21.653Z 2 Acked 1001 Write 5 Done Done Sent false
197842024-02-15T03:06:21.653Z 3 Acked 1002 Write 7 Done Done Sent false
197852024-02-15T03:06:21.653Z 4 Acked 1003 Flush 0 Done Done Sent false
197862024-02-15T03:06:21.653Z 5 Acked 1004 Write 1 Done Done Sent false
197872024-02-15T03:06:21.653Z 6 Acked 1005 Read 5 Done Done Sent false
197882024-02-15T03:06:21.654Z 7 Acked 1006 Write 9 Done Done Sent false
197892024-02-15T03:06:21.654Z 8 Acked 1007 Flush 0 Done Done Sent false
197902024-02-15T03:06:21.654Z 9 Acked 1008 Write 2 Done Done Sent false
197912024-02-15T03:06:21.654Z 10 Acked 1009 Read 10 Done Done Sent false
197922024-02-15T03:06:21.654Z 11 Acked 1010 Read 5 Done Done Sent false
197932024-02-15T03:06:21.654Z 12 Acked 1011 Read 2 Done Done Sent false
197942024-02-15T03:06:21.654Z 13 Acked 1012 Write 4 Done Done Sent false
197952024-02-15T03:06:21.654Z 14 Acked 1013 Write 8 Done Done Sent false
197962024-02-15T03:06:21.655Z 15 Acked 1014 Write 7 Done Done Sent false
197972024-02-15T03:06:21.655Z 16 Acked 1015 Write 7 Done Done Sent false
197982024-02-15T03:06:21.655Z 17 Acked 1016 Flush 0 Done Done Sent false
197992024-02-15T03:06:21.655Z 18 Acked 1017 Write 2 Done Done Sent false
198002024-02-15T03:06:21.655Z 19 Acked 1018 Read 6 Done Done Sent false
198012024-02-15T03:06:21.655Z 20 Acked 1019 Flush 0 Done Done Sent false
198022024-02-15T03:06:21.655Z 21 Acked 1020 Flush 0 Done Done Sent false
198032024-02-15T03:06:21.655Z 22 Acked 1021 Write 2 Done Done Sent false
198042024-02-15T03:06:21.655Z 23 Acked 1022 Read 1 Done Done Sent false
198052024-02-15T03:06:21.655Z 24 Acked 1023 Read 1 Done Done Sent false
198062024-02-15T03:06:21.656Z 25 Acked 1024 Write 2 Done Done Sent false
198072024-02-15T03:06:21.656Z 26 Acked 1025 Write 9 Done Done Sent false
198082024-02-15T03:06:21.656Z 27 Acked 1026 Read 9 Done Done Sent false
198092024-02-15T03:06:21.656Z 28 Acked 1027 Read 9 Done Done Sent false
198102024-02-15T03:06:21.656Z 29 Acked 1028 Read 3 Done Done Sent false
198112024-02-15T03:06:21.656Z 30 Acked 1029 Write 8 Sent Sent Sent false
198122024-02-15T03:06:21.656Z STATES DS:0 DS:1 DS:2 TOTAL
198132024-02-15T03:06:21.656Z New 0 0 0 0
198142024-02-15T03:06:21.656Z Sent 1 1 30 32
198152024-02-15T03:06:21.657Z Done 29 29 0 58
198162024-02-15T03:06:21.657Z Skipped 0 0 0 0
198172024-02-15T03:06:21.657Z Error 0 0 0 0
198182024-02-15T03:06:21.657ZLast Flush: 1020 1020 0
198192024-02-15T03:06:21.657ZDownstairs last five completed:
198202024-02-15T03:06:21.657ZUpstairs last five completed: 30 29 28 27 26
198212024-02-15T03:06:21.657Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:19.972753603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1856}
198222024-02-15T03:06:21.657ZWrote out file "/tmp/test_repair_verify.data"
198232024-02-15T03:06:21.657Z
198242024-02-15T03:06:21.658ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198252024-02-15T03:06:21.658ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198262024-02-15T03:06:21.658Z 0 000-019 178 178 175 295 295 287 F F F <---
198272024-02-15T03:06:21.658Z 1 020-039 169 169 169 277 277 277 F F F
198282024-02-15T03:06:21.658Z 2 040-059 175 175 175 287 287 287 F F F
198292024-02-15T03:06:21.658Z 3 060-079 178 178 177 294 294 290 T T F <---
198302024-02-15T03:06:21.658Z 4 080-099 178 178 177 294 294 290 T T F <---
198312024-02-15T03:06:21.659Z 5 100-119 173 173 173 285 285 285 F F F
198322024-02-15T03:06:21.659Z 6 120-139 177 177 177 290 290 290 F F F
198332024-02-15T03:06:21.659Z 7 140-159 177 177 177 290 290 290 T T F <---
198342024-02-15T03:06:21.659Z 8 160-179 170 170 170 282 282 282 F F F
198352024-02-15T03:06:21.659Z 9 180-199 175 175 175 287 287 287 F F F
198362024-02-15T03:06:21.659Z 10 200-219 178 178 177 294 294 290 F F F <---
198372024-02-15T03:06:21.659Z 11 220-239 178 178 177 294 294 290 F F F <---
198382024-02-15T03:06:21.659Z 12 240-259 173 173 173 285 285 285 F F F
198392024-02-15T03:06:21.660Z 13 260-279 178 178 177 293 293 290 F F F <---
198402024-02-15T03:06:21.660Z 14 280-299 178 178 177 294 294 290 F F F <---
198412024-02-15T03:06:21.660Z 15 300-319 176 176 176 289 289 289 F F F
198422024-02-15T03:06:21.660Z 16 320-339 178 178 161 294 294 263 F F F <---
198432024-02-15T03:06:21.660Z 17 340-359 175 175 175 287 287 287 F F F
198442024-02-15T03:06:21.661Z 18 360-379 178 178 169 292 292 277 F F F <---
198452024-02-15T03:06:21.661Z 19 380-399 166 166 166 274 274 274 F F F
198462024-02-15T03:06:21.661Z 20 400-419 176 176 176 288 288 288 T T F <---
198472024-02-15T03:06:21.661Z 21 420-439 177 177 177 290 290 290 F F F
198482024-02-15T03:06:21.661Z 22 440-459 177 177 177 290 290 290 F F F
198492024-02-15T03:06:21.661Z 23 460-479 178 178 175 293 293 287 F F F <---
198502024-02-15T03:06:21.662Z 24 480-499 166 166 166 274 274 274 F F F
198512024-02-15T03:06:21.662Z 25 500-519 173 173 173 285 285 285 F F F
198522024-02-15T03:06:21.662Z 26 520-539 178 178 177 294 294 290 F F F <---
198532024-02-15T03:06:21.662Z 27 540-559 177 177 177 290 290 290 F F F
198542024-02-15T03:06:21.662Z 28 560-579 177 177 177 290 290 290 F F F
198552024-02-15T03:06:21.662Z 29 580-599 175 175 175 287 287 287 F F F
198562024-02-15T03:06:21.662ZMax gen: 178, Max flush: 295
198572024-02-15T03:06:21.674ZError: Difference in extent metadata found!
198582024-02-15T03:06:21.674ZA Difference in extent metadata is expected here
198592024-02-15T03:06:21.674ZOn loop 89
198602024-02-15T03:06:21.674Z
198612024-02-15T03:06:21.674ZVerifying data now
198622024-02-15T03:06:25.769ZNo change after verify
198632024-02-15T03:06:25.770ZLoop: 89 Downstairs dump after verify (and repair):
198642024-02-15T03:06:26.208ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198652024-02-15T03:06:26.208Z 0 000-019 178 178 178 295 295 295 F F F
198662024-02-15T03:06:26.208Z 1 020-039 169 169 169 277 277 277 F F F
198672024-02-15T03:06:26.208Z 2 040-059 175 175 175 287 287 287 F F F
198682024-02-15T03:06:26.209Z 3 060-079 179 179 179 296 296 296 F F F
198692024-02-15T03:06:26.209Z 4 080-099 179 179 179 296 296 296 F F F
198702024-02-15T03:06:26.209Z 5 100-119 173 173 173 285 285 285 F F F
198712024-02-15T03:06:26.209Z 6 120-139 177 177 177 290 290 290 F F F
198722024-02-15T03:06:26.647Z 7 140-159 179 179 179 296 296 296 F F F
198732024-02-15T03:06:26.647Z 8 160-179 170 170 170 282 282 282 F F F
198742024-02-15T03:06:26.647Z 9 180-199 175 175 175 287 287 287 F F F
198752024-02-15T03:06:26.647Z 10 200-219 178 178 178 294 294 294 F F F
198762024-02-15T03:06:26.648Z 11 220-239 178 178 178 294 294 294 F F F
198772024-02-15T03:06:26.648Z 12 240-259 173 173 173 285 285 285 F F F
198782024-02-15T03:06:26.648Z 13 260-279 178 178 178 293 293 293 F F F
198792024-02-15T03:06:26.648Z 14 280-299 178 178 178 294 294 294 F F F
198802024-02-15T03:06:26.648Z 15 300-319 176 176 176 289 289 289 F F F
198812024-02-15T03:06:26.648Z 16 320-339 178 178 178 294 294 294 F F F
198822024-02-15T03:06:26.648Z 17 340-359 175 175 175 287 287 287 F F F
198832024-02-15T03:06:26.648Z 18 360-379 178 178 178 292 292 292 F F F
198842024-02-15T03:06:26.648Z 19 380-399 166 166 166 274 274 274 F F F
198852024-02-15T03:06:26.648Z 20 400-419 179 179 179 296 296 296 F F F
198862024-02-15T03:06:26.648Z 21 420-439 177 177 177 290 290 290 F F F
198872024-02-15T03:06:26.648Z 22 440-459 177 177 177 290 290 290 F F F
198882024-02-15T03:06:26.648Z 23 460-479 178 178 178 293 293 293 F F F
198892024-02-15T03:06:26.648Z 24 480-499 166 166 166 274 274 274 F F F
198902024-02-15T03:06:26.648Z 25 500-519 173 173 173 285 285 285 F F F
198912024-02-15T03:06:26.648Z 26 520-539 178 178 178 294 294 294 F F F
198922024-02-15T03:06:26.648Z 27 540-559 177 177 177 290 290 290 F F F
198932024-02-15T03:06:26.648Z 28 560-579 177 177 177 290 290 290 F F F
198942024-02-15T03:06:26.648Z 29 580-599 175 175 175 287 287 287 F F F
198952024-02-15T03:06:26.649ZMax gen: 179, Max flush: 296
198962024-02-15T03:06:26.649Z
198972024-02-15T03:06:26.649ZBegin loop 90
198982024-02-15T03:06:26.649ZDownstairs to restart: 0
198992024-02-15T03:06:26.649Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687238293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864}
199002024-02-15T03:06:26.649Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
199012024-02-15T03:06:26.649Z,"time":"2024-02-15T03:06:24.687541375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864}
199022024-02-15T03:06:26.649Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687588749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864}
199032024-02-15T03:06:26.649Z{"msg":"Crucible 58dacd2c-a0f1-45eb-9d88-3a81649dd40e has session id: 811e791a-4604-40a5-a165-cc713ddffde7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687621214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199042024-02-15T03:06:26.649Z{"msg":"Upstairs opts: Upstairs UUID: 58dacd2c-a0f1-45eb-9d88-3a81649dd40e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687649111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199052024-02-15T03:06:26.649Z{"msg":"Crucible stats registered with UUID: 58dacd2c-a0f1-45eb-9d88-3a81649dd40e","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687673168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199062024-02-15T03:06:26.649Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687687096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864}
199072024-02-15T03:06:26.649Z{"msg":"58dacd2c-a0f1-45eb-9d88-3a81649dd40e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687749798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199082024-02-15T03:06:26.649Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687780154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199092024-02-15T03:06:26.649Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687807041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199102024-02-15T03:06:26.649Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687834628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199112024-02-15T03:06:26.649Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687870863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199122024-02-15T03:06:26.649Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.68789579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199132024-02-15T03:06:26.649Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687938105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199142024-02-15T03:06:26.649Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.687980449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199152024-02-15T03:06:26.649Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688037902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199162024-02-15T03:06:26.649Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688078787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199172024-02-15T03:06:26.649Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:24.688111453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199182024-02-15T03:06:26.649Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:24.688142749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199192024-02-15T03:06:26.649Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:24.688181514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199202024-02-15T03:06:26.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688203052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199212024-02-15T03:06:26.650Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688247416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199222024-02-15T03:06:26.650Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688272023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199232024-02-15T03:06:26.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688306419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199242024-02-15T03:06:26.650Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688437872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199252024-02-15T03:06:26.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688905244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199262024-02-15T03:06:26.650Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688941139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199272024-02-15T03:06:26.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688958587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199282024-02-15T03:06:26.650Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.688999842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199292024-02-15T03:06:26.650Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.689510798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199302024-02-15T03:06:26.650Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.689659109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199312024-02-15T03:06:26.650Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.690125941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199322024-02-15T03:06:26.650Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.690177954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199332024-02-15T03:06:26.650Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.690224028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199342024-02-15T03:06:26.650Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.690259364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199352024-02-15T03:06:26.650Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.690275852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199362024-02-15T03:06:26.650Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:24.690298389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199372024-02-15T03:06:26.809Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.6899918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199382024-02-15T03:06:26.809Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.690104016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199392024-02-15T03:06:26.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.69087684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199402024-02-15T03:06:26.816Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.690916395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199412024-02-15T03:06:26.816Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.691412432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199422024-02-15T03:06:26.816Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692081078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199432024-02-15T03:06:26.816Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692122133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199442024-02-15T03:06:26.816Z{"msg":"[0]R flush_numbers[0..12]: [295, 277, 287, 296, 296, 285, 290, 296, 282, 287, 294, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692156879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199452024-02-15T03:06:26.816Z{"msg":"[0]R generation[0..12]: [178, 169, 175, 179, 179, 173, 177, 179, 170, 175, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692200094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199462024-02-15T03:06:26.816Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692223151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199472024-02-15T03:06:26.816Z{"msg":"[1]R flush_numbers[0..12]: [295, 277, 287, 296, 296, 285, 290, 296, 282, 287, 294, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692245458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199482024-02-15T03:06:26.816Z{"msg":"[1]R generation[0..12]: [178, 169, 175, 179, 179, 173, 177, 179, 170, 175, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692263906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199492024-02-15T03:06:26.816Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
199502024-02-15T03:06:26.816Z,"time":"2024-02-15T03:06:25.692282123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199512024-02-15T03:06:26.816Z{"msg":"[2]R flush_numbers[0..12]: [295, 277, 287, 296, 296, 285, 290, 296, 282, 287, 294, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692314339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199522024-02-15T03:06:26.816Z{"msg":"[2]R generation[0..12]: [178, 169, 175, 179, 179, 173, 177, 179, 170, 175, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692333217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199532024-02-15T03:06:26.816Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692365883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199542024-02-15T03:06:26.817Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.69238692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199552024-02-15T03:06:26.817Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692405148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199562024-02-15T03:06:26.817Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692434264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199572024-02-15T03:06:26.817Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692462601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"":"downstairs","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199582024-02-15T03:06:26.817Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692485228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199592024-02-15T03:06:26.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692512155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"0","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199602024-02-15T03:06:26.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692526233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"1","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199612024-02-15T03:06:26.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692552599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"client":"2","":"downstairs","session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199622024-02-15T03:06:26.817Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692568457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199632024-02-15T03:06:26.817Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692586995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199642024-02-15T03:06:26.817Z{"msg":"58dacd2c-a0f1-45eb-9d88-3a81649dd40e is now active with session: 811e791a-4604-40a5-a165-cc713ddffde7","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692601563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199652024-02-15T03:06:26.817Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.692613752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864,"session_id":"811e791a-4604-40a5-a165-cc713ddffde7"}
199662024-02-15T03:06:26.817Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:25.69262643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864}
199672024-02-15T03:06:26.817ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199682024-02-15T03:06:26.817ZLoading write count information from file "/tmp/test_repair_verify.data"
199692024-02-15T03:06:26.817ZRun Repair workload
199702024-02-15T03:06:26.817Z01/30 Write block 530 len 40960 data: 16 12 14 12 12 14 11 15 12 12
199712024-02-15T03:06:26.822Z02/30 Read block 508 len 24576
199722024-02-15T03:06:26.822Z03/30 Read block 394 len 12288
199732024-02-15T03:06:26.822Z04/30 Flush
199742024-02-15T03:06:26.822Z05/30 Write block 582 len 36864 data: 16 15 15 15 15 13 11 15 16
199752024-02-15T03:06:26.822Z06/30 Write block 510 len 20480 data: 15 13 15 12 15
199762024-02-15T03:06:26.822Z07/30 Write block 28 len 4096 data: 13
199772024-02-15T03:06:26.822Z08/30 Write block 161 len 12288 data: 12 15 14
199782024-02-15T03:06:26.833Z09/30 Write block 354 len 28672 data: 17 14 13 13 14 16 13
199792024-02-15T03:06:26.833Z10/30 Write block 582 len 8192 data: 17 16
199802024-02-15T03:06:26.833Z11/30 Write block 93 len 12288 data: 8 12 12
199812024-02-15T03:06:26.833Z12/30 Read block 369 len 28672
199822024-02-15T03:06:26.833Z13/30 Read block 480 len 36864
199832024-02-15T03:06:26.833Z14/30 Write block 389 len 12288 data: 17 15 14
199842024-02-15T03:06:26.833Z15/30 Read block 98 len 40960
199852024-02-15T03:06:26.833Z16/30 Read block 328 len 24576
199862024-02-15T03:06:26.833Z17/30 Write block 408 len 8192 data: 18 24
199872024-02-15T03:06:26.833Z18/30 Read block 196 len 24576
199882024-02-15T03:06:26.837Z19/30 Flush
199892024-02-15T03:06:26.837Z20/30 Write block 10 len 24576 data: 16 12 10 9 9 10
199902024-02-15T03:06:26.837Z21/30 Read block 95 len 4096
199912024-02-15T03:06:26.837Z22/30 Write block 332 len 32768 data: 16 16 19 18 16 16 16 12
199922024-02-15T03:06:26.837Z23/30 Write block 205 len 40960 data: 13 11 12 14 14 10 12 13 13 15
199932024-02-15T03:06:26.844Z24/30 Read block 128 len 24576
199942024-02-15T03:06:26.844Z25/30 Read block 340 len 24576
199952024-02-15T03:06:26.844Z26/30 Read block 55 len 40960
199962024-02-15T03:06:26.844Z27/30 Read block 1 len 20480
199972024-02-15T03:06:26.844Z28/30 Read block 553 len 36864
199982024-02-15T03:06:26.844Z29/30 Read block 513 len 32768
199992024-02-15T03:06:26.844Z30/30 Read block 328 len 16384
200002024-02-15T03:06:26.852Z----------------------------------------------------------------
200012024-02-15T03:06:26.852Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
200022024-02-15T03:06:26.852ZStates: Active Active Active
200032024-02-15T03:06:26.852ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200042024-02-15T03:06:26.852Z 1 Acked 1000 Write 10 Sent Done Done false
200052024-02-15T03:06:26.852Z 2 Acked 1001 Read 6 Sent Done Done false
200062024-02-15T03:06:26.852Z 3 Acked 1002 Read 3 Sent Done Done false
200072024-02-15T03:06:26.852Z 4 Acked 1003 Flush 0 Sent Done Done false
200082024-02-15T03:06:26.852Z 5 Acked 1004 Write 9 Sent Done Done false
200092024-02-15T03:06:26.853Z 6 Acked 1005 Write 5 Sent Done Done false
200102024-02-15T03:06:26.853Z 7 Acked 1006 Write 1 Sent Done Done false
200112024-02-15T03:06:26.853Z 8 Acked 1007 Write 3 Sent Done Done false
200122024-02-15T03:06:26.853Z 9 Acked 1008 Write 7 Sent Done Done false
200132024-02-15T03:06:26.853Z 10 Acked 1009 Write 2 Sent Done Done false
200142024-02-15T03:06:26.853Z 11 Acked 1010 Write 3 Sent Done Done false
200152024-02-15T03:06:26.853Z 12 Acked 1011 Read 7 Sent Done Done false
200162024-02-15T03:06:26.853Z 13 Acked 1012 Read 9 Sent Done Done false
200172024-02-15T03:06:26.853Z 14 Acked 1013 Write 3 Sent Done Done false
200182024-02-15T03:06:26.853Z 15 Acked 1014 Read 10 Sent Done Done false
200192024-02-15T03:06:26.853Z 16 Acked 1015 Read 6 Sent Done Done false
200202024-02-15T03:06:26.853Z 17 Acked 1016 Write 2 Sent Done Done false
200212024-02-15T03:06:26.853Z 18 Acked 1017 Read 6 Sent Done Done false
200222024-02-15T03:06:26.853Z 19 Acked 1018 Flush 0 Sent Done Done false
200232024-02-15T03:06:26.853Z 20 Acked 1019 Write 6 Sent Done Done false
200242024-02-15T03:06:26.853Z 21 Acked 1020 Read 1 Sent Done Done false
200252024-02-15T03:06:26.853Z 22 Acked 1021 Write 8 Sent Done Done false
200262024-02-15T03:06:26.853Z 23 Acked 1022 Write 10 Sent Done Done false
200272024-02-15T03:06:26.853Z 24 Acked 1023 Read 6 Sent Done Done false
200282024-02-15T03:06:26.853Z 25 Acked 1024 Read 6 Sent Done Done false
200292024-02-15T03:06:26.853Z 26 Acked 1025 Read 10 Sent Done Done false
200302024-02-15T03:06:26.853Z 27 Acked 1026 Read 5 Sent Done Done false
200312024-02-15T03:06:26.853Z 28 Acked 1027 Read 9 Sent Done Done false
200322024-02-15T03:06:26.853Z 29 Acked 1028 Read 8 Sent Done Done false
200332024-02-15T03:06:26.854Z 30 Acked 1029 Read 4 Sent Done Done false
200342024-02-15T03:06:26.854Z STATES DS:0 DS:1 DS:2 TOTAL
200352024-02-15T03:06:26.854Z New 0 0 0 0
200362024-02-15T03:06:26.854Z Sent 30 0 0 30
200372024-02-15T03:06:26.854Z Done 0 30 30 60
200382024-02-15T03:06:26.854Z Skipped 0 0 0 0
200392024-02-15T03:06:26.854Z Error 0 0 0 0
200402024-02-15T03:06:26.854ZLast Flush: 0 1018 1018
200412024-02-15T03:06:26.854ZDownstairs last five completed:
200422024-02-15T03:06:26.854ZUpstairs last five completed: 30 29 28 27 26
200432024-02-15T03:06:26.854Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:25.726477539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1864}
200442024-02-15T03:06:26.854ZWrote out file "/tmp/test_repair_verify.data"
200452024-02-15T03:06:26.854Z
200462024-02-15T03:06:26.865ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200472024-02-15T03:06:26.875ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200482024-02-15T03:06:26.875Z 0 000-019 178 178 178 295 295 295 F T T <---
200492024-02-15T03:06:26.875Z 1 020-039 169 180 180 277 298 298 F F F <---
200502024-02-15T03:06:26.875Z 2 040-059 175 175 175 287 287 287 F F F
200512024-02-15T03:06:26.875Z 3 060-079 179 179 179 296 296 296 F F F
200522024-02-15T03:06:26.875Z 4 080-099 179 180 180 296 298 298 F F F <---
200532024-02-15T03:06:26.875Z 5 100-119 173 173 173 285 285 285 F F F
200542024-02-15T03:06:26.875Z 6 120-139 177 177 177 290 290 290 F F F
200552024-02-15T03:06:26.875Z 7 140-159 179 179 179 296 296 296 F F F
200562024-02-15T03:06:26.875Z 8 160-179 170 180 180 282 298 298 F F F <---
200572024-02-15T03:06:26.875Z 9 180-199 175 175 175 287 287 287 F F F
200582024-02-15T03:06:26.875Z 10 200-219 178 178 178 294 294 294 F T T <---
200592024-02-15T03:06:26.875Z 11 220-239 178 178 178 294 294 294 F F F
200602024-02-15T03:06:26.875Z 12 240-259 173 173 173 285 285 285 F F F
200612024-02-15T03:06:26.875Z 13 260-279 178 178 178 293 293 293 F F F
200622024-02-15T03:06:26.875Z 14 280-299 178 178 178 294 294 294 F F F
200632024-02-15T03:06:26.876Z 15 300-319 176 176 176 289 289 289 F F F
200642024-02-15T03:06:26.876Z 16 320-339 178 178 178 294 294 294 F T T <---
200652024-02-15T03:06:26.876Z 17 340-359 175 180 180 287 298 298 F F F <---
200662024-02-15T03:06:26.876Z 18 360-379 178 180 180 292 298 298 F F F <---
200672024-02-15T03:06:26.876Z 19 380-399 166 180 180 274 298 298 F F F <---
200682024-02-15T03:06:26.876Z 20 400-419 179 180 180 296 298 298 F F F <---
200692024-02-15T03:06:26.876Z 21 420-439 177 177 177 290 290 290 F F F
200702024-02-15T03:06:26.876Z 22 440-459 177 177 177 290 290 290 F F F
200712024-02-15T03:06:26.876Z 23 460-479 178 178 178 293 293 293 F F F
200722024-02-15T03:06:26.876Z 24 480-499 166 166 166 274 274 274 F F F
200732024-02-15T03:06:26.876Z 25 500-519 173 180 180 285 298 298 F F F <---
200742024-02-15T03:06:26.876Z 26 520-539 178 180 180 294 297 297 F F F <---
200752024-02-15T03:06:26.876Z 27 540-559 177 177 177 290 290 290 F F F
200762024-02-15T03:06:26.876Z 28 560-579 177 177 177 290 290 290 F F F
200772024-02-15T03:06:26.876Z 29 580-599 175 180 180 287 298 298 F F F <---
200782024-02-15T03:06:26.876ZMax gen: 180, Max flush: 298
200792024-02-15T03:06:26.876ZError: Difference in extent metadata found!
200802024-02-15T03:06:26.976ZA Difference in extent metadata is expected here
200812024-02-15T03:06:26.976ZOn loop 90
200822024-02-15T03:06:26.976Z
200832024-02-15T03:06:26.976ZVerifying data now
200842024-02-15T03:06:31.524ZNo change after verify
200852024-02-15T03:06:31.524ZLoop: 90 Downstairs dump after verify (and repair):
200862024-02-15T03:06:31.541ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200872024-02-15T03:06:31.541Z 0 000-019 181 181 181 299 299 299 F F F
200882024-02-15T03:06:31.541Z 1 020-039 180 180 180 298 298 298 F F F
200892024-02-15T03:06:31.541Z 2 040-059 175 175 175 287 287 287 F F F
200902024-02-15T03:06:31.541Z 3 060-079 179 179 179 296 296 296 F F F
200912024-02-15T03:06:31.542Z 4 080-099 180 180 180 298 298 298 F F F
200922024-02-15T03:06:31.542Z 5 100-119 173 173 173 285 285 285 F F F
200932024-02-15T03:06:31.542Z 6 120-139 177 177 177 290 290 290 F F F
200942024-02-15T03:06:31.542Z 7 140-159 179 179 179 296 296 296 F F F
200952024-02-15T03:06:31.542Z 8 160-179 180 180 180 298 298 298 F F F
200962024-02-15T03:06:31.542Z 9 180-199 175 175 175 287 287 287 F F F
200972024-02-15T03:06:31.542Z 10 200-219 181 181 181 299 299 299 F F F
200982024-02-15T03:06:31.542Z 11 220-239 178 178 178 294 294 294 F F F
200992024-02-15T03:06:31.542Z 12 240-259 173 173 173 285 285 285 F F F
201002024-02-15T03:06:31.542Z 13 260-279 178 178 178 293 293 293 F F F
201012024-02-15T03:06:31.542Z 14 280-299 178 178 178 294 294 294 F F F
201022024-02-15T03:06:31.542Z 15 300-319 176 176 176 289 289 289 F F F
201032024-02-15T03:06:31.542Z 16 320-339 181 181 181 299 299 299 F F F
201042024-02-15T03:06:31.546Z 17 340-359 180 180 180 298 298 298 F F F
201052024-02-15T03:06:31.546Z 18 360-379 180 180 180 298 298 298 F F F
201062024-02-15T03:06:31.546Z 19 380-399 180 180 180 298 298 298 F F F
201072024-02-15T03:06:31.546Z 20 400-419 180 180 180 298 298 298 F F F
201082024-02-15T03:06:31.546Z 21 420-439 177 177 177 290 290 290 F F F
201092024-02-15T03:06:31.546Z 22 440-459 177 177 177 290 290 290 F F F
201102024-02-15T03:06:31.546Z 23 460-479 178 178 178 293 293 293 F F F
201112024-02-15T03:06:31.546Z 24 480-499 166 166 166 274 274 274 F F F
201122024-02-15T03:06:31.546Z 25 500-519 180 180 180 298 298 298 F F F
201132024-02-15T03:06:31.547Z 26 520-539 180 180 180 297 297 297 F F F
201142024-02-15T03:06:31.547Z 27 540-559 177 177 177 290 290 290 F F F
201152024-02-15T03:06:31.547Z 28 560-579 177 177 177 290 290 290 F F F
201162024-02-15T03:06:31.547Z 29 580-599 180 180 180 298 298 298 F F F
201172024-02-15T03:06:31.547ZMax gen: 181, Max flush: 299
201182024-02-15T03:06:31.547Z
201192024-02-15T03:06:31.547ZBegin loop 91
201202024-02-15T03:06:31.547ZDownstairs to restart: 1
201212024-02-15T03:06:31.564Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444324387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872}
201222024-02-15T03:06:31.565Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444664594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872}
201232024-02-15T03:06:31.570Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444713438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872}
201242024-02-15T03:06:31.570Z{"msg":"Crucible fbc43df9-3160-4c42-b839-6f650d4fd342 has session id: c111ab64-2085-4098-8b17-74ba360463e2","v":0Crucible runtime is spawned
201252024-02-15T03:06:31.571Z,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444753393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201262024-02-15T03:06:31.571Z{"msg":"Upstairs opts: Upstairs UUID: fbc43df9-3160-4c42-b839-6f650d4fd342, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444798368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201272024-02-15T03:06:31.571Z{"msg":"Crucible stats registered with UUID: fbc43df9-3160-4c42-b839-6f650d4fd342","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444843602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201282024-02-15T03:06:31.571Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444873498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872}
201292024-02-15T03:06:31.571Z{"msg":"fbc43df9-3160-4c42-b839-6f650d4fd342 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.444964177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201302024-02-15T03:06:31.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445003622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201312024-02-15T03:06:31.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445029489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201322024-02-15T03:06:31.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445058685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201332024-02-15T03:06:31.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445077183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201342024-02-15T03:06:31.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445093971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201352024-02-15T03:06:31.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445122887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201362024-02-15T03:06:31.571Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445155993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201372024-02-15T03:06:31.571Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445201937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201382024-02-15T03:06:31.572Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445318532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201392024-02-15T03:06:31.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445370686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201402024-02-15T03:06:31.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445399073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201412024-02-15T03:06:31.572Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:30.445441427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201422024-02-15T03:06:31.572Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:30.445484132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201432024-02-15T03:06:31.572Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:30.445525546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201442024-02-15T03:06:31.572Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445567251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201452024-02-15T03:06:31.572Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445604757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201462024-02-15T03:06:31.572Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.445628014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201472024-02-15T03:06:31.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.446048851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201482024-02-15T03:06:31.572Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.446084707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201492024-02-15T03:06:31.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.446101005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201502024-02-15T03:06:31.573Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.44613213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201512024-02-15T03:06:31.573Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.446664574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201522024-02-15T03:06:31.573Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.446871968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201532024-02-15T03:06:31.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.44717249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201542024-02-15T03:06:31.573Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.447209675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201552024-02-15T03:06:31.573Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.447226133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201562024-02-15T03:06:31.573Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.447424939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201572024-02-15T03:06:31.573Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.447472803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201582024-02-15T03:06:31.573Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:30.447507798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201592024-02-15T03:06:32.569Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.447385567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201602024-02-15T03:06:32.569Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.447551716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201612024-02-15T03:06:32.582Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.448513495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201622024-02-15T03:06:32.582Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.448553841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201632024-02-15T03:06:32.582Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449073435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201642024-02-15T03:06:32.582Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449636845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201652024-02-15T03:06:32.582Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449667161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201662024-02-15T03:06:32.583Z{"msg":"[0]R flush_numbers[0..12]: [299, 298, 287, 296, 298, 285, 290, 296, 298, 287, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449724044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201672024-02-15T03:06:32.583Z{"msg":"[0]R generation[0..12]: [181, 180, 175, 179, 180, 173, 177, 179, 180, 175, 181, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449774528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201682024-02-15T03:06:32.583Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449812343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201692024-02-15T03:06:32.583Z{"msg":"[1]R flush_numbers[0..12]: [299, 298, 287, 296, 298, 285, 290, 296, 298, 287, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449849898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201702024-02-15T03:06:32.583Z{"msgWait for a query_work_queue command to finish before sending IO
201712024-02-15T03:06:32.583Z":"[1]R generation[0..12]: [181, 180, 175, 179, 180, 173, 177, 179, 180, 175, 181, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449883554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201722024-02-15T03:06:32.583Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449923379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201732024-02-15T03:06:32.583Z{"msg":"[2]R flush_numbers[0..12]: [299, 298, 287, 296, 298, 285, 290, 296, 298, 287, 299, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449946896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201742024-02-15T03:06:32.583Z{"msg":"[2]R generation[0..12]: [181, 180, 175, 179, 180, 173, 177, 179, 180, 175, 181, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.449978172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201752024-02-15T03:06:32.583Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450012348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201762024-02-15T03:06:32.583Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450042374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201772024-02-15T03:06:32.583Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450060352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201782024-02-15T03:06:32.584Z{"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450090348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201792024-02-15T03:06:32.584Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450109406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"":"downstairs","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201802024-02-15T03:06:32.584Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450139062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201812024-02-15T03:06:32.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.45015523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"0","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201822024-02-15T03:06:32.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450186856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"1","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201832024-02-15T03:06:32.584Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450205543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"client":"2","":"downstairs","session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201842024-02-15T03:06:32.584Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.45023545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201852024-02-15T03:06:32.584Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450252518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201862024-02-15T03:06:32.584Z{"msg":"fbc43df9-3160-4c42-b839-6f650d4fd342 is now active with session: c111ab64-2085-4098-8b17-74ba360463e2","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450281734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201872024-02-15T03:06:32.584Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450298522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872,"session_id":"c111ab64-2085-4098-8b17-74ba360463e2"}
201882024-02-15T03:06:32.584Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:31.450327488Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201892024-02-15T03:06:32.584Zip-10-150-1-55.us-west-2.compute.internal","pid":1872}
201902024-02-15T03:06:32.584ZLoading write count information from file "/tmp/test_repair_verify.data"
201912024-02-15T03:06:32.585ZRun Repair workload
201922024-02-15T03:06:32.585Z01/30 Write block 264 len 16384 data: 12 14 12 8
201932024-02-15T03:06:32.585Z02/30 Write block 299 len 8192 data: 14 13
201942024-02-15T03:06:32.585Z03/30 Write block 244 len 8192 data: 13 13
201952024-02-15T03:06:32.585Z04/30 Write block 483 len 4096 data: 11
201962024-02-15T03:06:32.585Z05/30 Write block 374 len 36864 data: 12 12 13 15 15 13 12 13 12
201972024-02-15T03:06:32.585Z06/30 Read block 80 len 4096
201982024-02-15T03:06:32.585Z07/30 Read block 567 len 24576
201992024-02-15T03:06:32.590Z08/30 Write block 185 len 8192 data: 19 18
202002024-02-15T03:06:32.590Z09/30 Write block 157 len 24576 data: 19 15 12 10 13 16
202012024-02-15T03:06:32.590Z10/30 Read block 503 len 20480
202022024-02-15T03:06:32.590Z11/30 Write block 593 len 12288 data: 15 14 13
202032024-02-15T03:06:32.590Z12/30 Write block 23 len 28672 data: 5 8 8 9 11 14 12
202042024-02-15T03:06:32.590Z13/30 Read block 269 len 20480
202052024-02-15T03:06:32.590Z14/30 Read block 92 len 24576
202062024-02-15T03:06:32.590Z15/30 Read block 497 len 40960
202072024-02-15T03:06:32.590Z16/30 Write block 504 len 40960 data: 17 16 16 12 12 16 16 14 16 13
202082024-02-15T03:06:32.590Z17/30 Read block 102 len 16384
202092024-02-15T03:06:32.590Z18/30 Read block 278 len 8192
202102024-02-15T03:06:32.591Z19/30 Write block 300 len 4096 data: 14
202112024-02-15T03:06:32.591Z20/30 Read block 408 len 40960
202122024-02-15T03:06:32.591Z21/30 Read block 498 len 32768
202132024-02-15T03:06:32.591Z22/30 Read block 415 len 12288
202142024-02-15T03:06:32.600Z23/30 Read block 307 len 28672
202152024-02-15T03:06:32.601Z24/30 Write block 214 len 8192 data: 16 15
202162024-02-15T03:06:32.601Z25/30 Write block 569 len 20480 data: 13 15 15 15 18
202172024-02-15T03:06:32.601Z26/30 Read block 176 len 20480
202182024-02-15T03:06:32.601Z27/30 Read block 48 len 32768
202192024-02-15T03:06:32.601Z28/30 Read block 11 len 40960
202202024-02-15T03:06:32.601Z29/30 Read block 170 len 12288
202212024-02-15T03:06:32.602Z30/30 Write block 278 len 32768 data: 12 14 14 13 11 14 16 16
202222024-02-15T03:06:32.602Z----------------------------------------------------------------
202232024-02-15T03:06:32.602Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
202242024-02-15T03:06:32.602ZStates: Active Active Active
202252024-02-15T03:06:32.602ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202262024-02-15T03:06:32.603Z 1 Acked 1000 Write 4 Done Sent Done false
202272024-02-15T03:06:32.603Z 2 Acked 1001 Write 2 Done Sent Done false
202282024-02-15T03:06:32.603Z 3 Acked 1002 Write 2 Done Sent Done false
202292024-02-15T03:06:32.603Z 4 Acked 1003 Write 1 Done Sent Done false
202302024-02-15T03:06:32.603Z 5 Acked 1004 Write 9 Done Sent Done false
202312024-02-15T03:06:32.603Z 6 Acked 1005 Read 1 Done Sent Done false
202322024-02-15T03:06:32.604Z 7 Acked 1006 Read 6 Done Sent Done false
202332024-02-15T03:06:32.604Z 8 Acked 1007 Write 2 Done Sent Done false
202342024-02-15T03:06:32.604Z 9 Acked 1008 Write 6 Done Sent Done false
202352024-02-15T03:06:32.604Z 10 Acked 1009 Read 5 Done Sent Done false
202362024-02-15T03:06:32.604Z 11 Acked 1010 Write 3 Done Sent Done false
202372024-02-15T03:06:32.604Z 12 Acked 1011 Write 7 Done Sent Done false
202382024-02-15T03:06:32.604Z 13 Acked 1012 Read 5 Done Sent Done false
202392024-02-15T03:06:32.604Z 14 Acked 1013 Read 6 Done Sent Done false
202402024-02-15T03:06:32.604Z 15 Acked 1014 Read 10 Done Sent Done false
202412024-02-15T03:06:32.604Z 16 Acked 1015 Write 10 Done Sent Done false
202422024-02-15T03:06:32.604Z 17 Acked 1016 Read 4 Done Sent Done false
202432024-02-15T03:06:32.604Z 18 Acked 1017 Read 2 Done Sent Done false
202442024-02-15T03:06:32.604Z 19 Acked 1018 Write 1 Done Sent Done false
202452024-02-15T03:06:32.605Z 20 Acked 1019 Read 10 Done Sent Done false
202462024-02-15T03:06:32.605Z 21 Acked 1020 Read 8 Done Sent Done false
202472024-02-15T03:06:32.605Z 22 Acked 1021 Read 3 Done Sent Done false
202482024-02-15T03:06:32.605Z 23 Acked 1022 Read 7 Done Sent Done false
202492024-02-15T03:06:32.605Z 24 Acked 1023 Write 2 Done Sent Done false
202502024-02-15T03:06:32.605Z 25 Acked 1024 Write 5 Done Sent Done false
202512024-02-15T03:06:32.605Z 26 Acked 1025 Read 5 Done Sent Done false
202522024-02-15T03:06:32.605Z 27 Acked 1026 Read 8 Done Sent Done false
202532024-02-15T03:06:32.605Z 28 Acked 1027 Read 10 Done Sent Done false
202542024-02-15T03:06:32.605Z 29 Acked 1028 Read 3 Done Sent Done false
202552024-02-15T03:06:32.605Z 30 Acked 1029 Write 8 Sent Sent Sent false
202562024-02-15T03:06:32.605Z STATES DS:0 DS:1 DS:2 TOTAL
202572024-02-15T03:06:32.605Z New 0 0 0 0
202582024-02-15T03:06:32.606Z Sent 1 30 1 32
202592024-02-15T03:06:32.606Z Done 29 0 29 58
202602024-02-15T03:06:32.606Z Skipped 0 0 0 0
202612024-02-15T03:06:32.606Z Error 0 0 0 0
202622024-02-15T03:06:32.606ZLast Flush: 0 0 0
202632024-02-15T03:06:32.606ZDownstairs last five completed:
202642024-02-15T03:06:32.628ZUpstairs last five completed: 30 29 28 27 26
202652024-02-15T03:06:32.628Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:31.47697392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1872}
202662024-02-15T03:06:32.628ZWrote out file "/tmp/test_repair_verify.data"
202672024-02-15T03:06:32.628Z
202682024-02-15T03:06:32.628ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202692024-02-15T03:06:32.635ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202702024-02-15T03:06:32.635Z 0 000-019 181 181 181 299 299 299 F F F
202712024-02-15T03:06:32.635Z 1 020-039 180 180 180 298 298 298 T F T <---
202722024-02-15T03:06:32.636Z 2 040-059 175 175 175 287 287 287 F F F
202732024-02-15T03:06:32.636Z 3 060-079 179 179 179 296 296 296 F F F
202742024-02-15T03:06:32.636Z 4 080-099 180 180 180 298 298 298 F F F
202752024-02-15T03:06:32.636Z 5 100-119 173 173 173 285 285 285 F F F
202762024-02-15T03:06:32.636Z 6 120-139 177 177 177 290 290 290 F F F
202772024-02-15T03:06:32.636Z 7 140-159 179 179 179 296 296 296 T F T <---
202782024-02-15T03:06:32.636Z 8 160-179 180 180 180 298 298 298 T F T <---
202792024-02-15T03:06:32.636Z 9 180-199 175 175 175 287 287 287 T F T <---
202802024-02-15T03:06:32.636Z 10 200-219 181 181 181 299 299 299 T F T <---
202812024-02-15T03:06:32.636Z 11 220-239 178 178 178 294 294 294 F F F
202822024-02-15T03:06:32.636Z 12 240-259 173 173 173 285 285 285 T F T <---
202832024-02-15T03:06:32.636Z 13 260-279 178 178 178 293 293 293 T F T <---
202842024-02-15T03:06:32.636Z 14 280-299 178 178 178 294 294 294 T F T <---
202852024-02-15T03:06:32.637Z 15 300-319 176 176 176 289 289 289 T F T <---
202862024-02-15T03:06:32.637Z 16 320-339 181 181 181 299 299 299 F F F
202872024-02-15T03:06:32.637Z 17 340-359 180 180 180 298 298 298 F F F
202882024-02-15T03:06:32.637Z 18 360-379 180 180 180 298 298 298 T F T <---
202892024-02-15T03:06:32.637Z 19 380-399 180 180 180 298 298 298 T F T <---
202902024-02-15T03:06:32.637Z 20 400-419 180 180 180 298 298 298 F F F
202912024-02-15T03:06:32.637Z 21 420-439 177 177 177 290 290 290 F F F
202922024-02-15T03:06:32.637Z 22 440-459 177 177 177 290 290 290 F F F
202932024-02-15T03:06:32.637Z 23 460-479 178 178 178 293 293 293 F F F
202942024-02-15T03:06:32.637Z 24 480-499 166 166 166 274 274 274 T F T <---
202952024-02-15T03:06:32.637Z 25 500-519 180 180 180 298 298 298 T F T <---
202962024-02-15T03:06:32.637Z 26 520-539 180 180 180 297 297 297 F F F
202972024-02-15T03:06:32.638Z 27 540-559 177 177 177 290 290 290 F F F
202982024-02-15T03:06:32.638Z 28 560-579 177 177 177 290 290 290 T F T <---
202992024-02-15T03:06:32.638Z 29 580-599 180 180 180 298 298 298 T F T <---
203002024-02-15T03:06:32.638ZMax gen: 181, Max flush: 299
203012024-02-15T03:06:32.638ZError: Difference in extent metadata found!
203022024-02-15T03:06:32.638ZA Difference in extent metadata is expected here
203032024-02-15T03:06:32.638ZOn loop 91
203042024-02-15T03:06:32.638Z
203052024-02-15T03:06:32.654ZVerifying data now
203062024-02-15T03:06:37.406ZNo change after verify
203072024-02-15T03:06:37.406ZLoop: 91 Downstairs dump after verify (and repair):
203082024-02-15T03:06:37.422ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203092024-02-15T03:06:37.422Z 0 000-019 181 181 181 299 299 299 F F F
203102024-02-15T03:06:37.422Z 1 020-039 182 182 182 300 300 300 F F F
203112024-02-15T03:06:37.422Z 2 040-059 175 175 175 287 287 287 F F F
203122024-02-15T03:06:37.422Z 3 060-079 179 179 179 296 296 296 F F F
203132024-02-15T03:06:37.423Z 4 080-099 180 180 180 298 298 298 F F F
203142024-02-15T03:06:37.423Z 5 100-119 173 173 173 285 285 285 F F F
203152024-02-15T03:06:37.423Z 6 120-139 177 177 177 290 290 290 F F F
203162024-02-15T03:06:37.423Z 7 140-159 182 182 182 300 300 300 F F F
203172024-02-15T03:06:37.423Z 8 160-179 182 182 182 300 300 300 F F F
203182024-02-15T03:06:37.423Z 9 180-199 182 182 182 300 300 300 F F F
203192024-02-15T03:06:37.423Z 10 200-219 182 182 182 300 300 300 F F F
203202024-02-15T03:06:37.423Z 11 220-239 178 178 178 294 294 294 F F F
203212024-02-15T03:06:37.424Z 12 240-259 182 182 182 300 300 300 F F F
203222024-02-15T03:06:37.424Z 13 260-279 182 182 182 300 300 300 F F F
203232024-02-15T03:06:37.424Z 14 280-299 182 182 182 300 300 300 F F F
203242024-02-15T03:06:37.424Z 15 300-319 182 182 182 300 300 300 F F F
203252024-02-15T03:06:37.424Z 16 320-339 181 181 181 299 299 299 F F F
203262024-02-15T03:06:37.425Z 17 340-359 180 180 180 298 298 298 F F F
203272024-02-15T03:06:37.425Z 18 360-379 182 182 182 300 300 300 F F F
203282024-02-15T03:06:37.425Z 19 380-399 182 182 182 300 300 300 F F F
203292024-02-15T03:06:37.425Z 20 400-419 180 180 180 298 298 298 F F F
203302024-02-15T03:06:37.425Z 21 420-439 177 177 177 290 290 290 F F F
203312024-02-15T03:06:37.425Z 22 440-459 177 177 177 290 290 290 F F F
203322024-02-15T03:06:37.425Z 23 460-479 178 178 178 293 293 293 F F F
203332024-02-15T03:06:37.425Z 24 480-499 182 182 182 300 300 300 F F F
203342024-02-15T03:06:37.425Z 25 500-519 182 182 182 300 300 300 F F F
203352024-02-15T03:06:37.425Z 26 520-539 180 180 180 297 297 297 F F F
203362024-02-15T03:06:37.426Z 27 540-559 177 177 177 290 290 290 F F F
203372024-02-15T03:06:37.426Z 28 560-579 182 182 182 300 300 300 F F F
203382024-02-15T03:06:37.426Z 29 580-599 182 182 182 300 300 300 F F F
203392024-02-15T03:06:37.426ZMax gen: 182, Max flush: 300
203402024-02-15T03:06:37.449Z
203412024-02-15T03:06:37.449ZBegin loop 92
203422024-02-15T03:06:37.449ZDownstairs to restart: 1
203432024-02-15T03:06:37.449Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311378543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880}
203442024-02-15T03:06:37.449Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311677326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880}
203452024-02-15T03:06:37.449Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311712781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880}
203462024-02-15T03:06:37.449Z{"msg":"Crucible a54802b6-7a73-433c-a89a-db1a3f4a97cc has session id: abb4ed18-396b-45f6-a826-22a2d6ace6f5","v":0,"name":"crucible","level":30Crucible runtime is spawned
203472024-02-15T03:06:37.449Z,"time":"2024-02-15T03:06:36.311743987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203482024-02-15T03:06:37.449Z{"msg":"Upstairs opts: Upstairs UUID: a54802b6-7a73-433c-a89a-db1a3f4a97cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311791081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203492024-02-15T03:06:37.450Z{"msg":"Crucible stats registered with UUID: a54802b6-7a73-433c-a89a-db1a3f4a97cc","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311818138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203502024-02-15T03:06:37.450Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311832056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880}
203512024-02-15T03:06:37.450Z{"msg":"a54802b6-7a73-433c-a89a-db1a3f4a97cc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311972269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203522024-02-15T03:06:37.450Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.311997525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203532024-02-15T03:06:37.450Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312012924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203542024-02-15T03:06:37.450Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312027002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203552024-02-15T03:06:37.450Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312045799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203562024-02-15T03:06:37.450Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312062277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203572024-02-15T03:06:37.450Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312083835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203582024-02-15T03:06:37.451Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312138308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203592024-02-15T03:06:37.451Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312175163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203602024-02-15T03:06:37.451Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312192281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203612024-02-15T03:06:37.451Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:36.312369799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203622024-02-15T03:06:37.451Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:36.312468027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203632024-02-15T03:06:37.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312513821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203642024-02-15T03:06:37.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312557035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203652024-02-15T03:06:37.451Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:36.312608249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203662024-02-15T03:06:37.451Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312655813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203672024-02-15T03:06:37.452Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312682989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203682024-02-15T03:06:37.452Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.312710336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203692024-02-15T03:06:37.466Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.313082849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203702024-02-15T03:06:37.467Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.313131173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203712024-02-15T03:06:37.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.31315909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203722024-02-15T03:06:37.467Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.313199395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203732024-02-15T03:06:37.467Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.313766564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203742024-02-15T03:06:37.467Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.313816518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203752024-02-15T03:06:37.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.314389296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203762024-02-15T03:06:37.467Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.31443789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203772024-02-15T03:06:37.468Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.314466556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203782024-02-15T03:06:37.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.314505171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203792024-02-15T03:06:37.468Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.314529168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203802024-02-15T03:06:37.468Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:36.314561864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203812024-02-15T03:06:38.439Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.314281302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203822024-02-15T03:06:38.440Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.314400157Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203832024-02-15T03:06:38.489Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.315361557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203842024-02-15T03:06:38.489Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.315404041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203852024-02-15T03:06:38.490Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.315921686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203862024-02-15T03:06:38.490Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316475467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203872024-02-15T03:06:38.490Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316510073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203882024-02-15T03:06:38.490Z{"msg":"[0]R flush_numbers[0..12]: [299, 300, 287, 296, 298, 285, 290, 300, 300, 300, 300, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316527681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203892024-02-15T03:06:38.490Z{"msg":"[0]R generation[0..12]: [181, 182, 175, 179, 180, 173, 177, 182, 182, 182, 182, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316555567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203902024-02-15T03:06:38.490Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316578444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203912024-02-15T03:06:38.490Z{"msg":"[1]R flush_numbers[0..12]: [299, 300, 287, 296, 298, 285, 290, 300, 300, 300, 300, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316605191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203922024-02-15T03:06:38.490Z{"msg":"[1]R generation[0..12]: [181, 182, 175, 179, 180, 173, 177, 182, 182, 182, 182, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316620339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203932024-02-15T03:06:38.490Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316645846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203942024-02-15T03:06:38.490Z{"msg":"Wait for a query_work_queue command to finish before sending IO
203952024-02-15T03:06:38.490Z[2]R flush_numbers[0..12]: [299, 300, 287, 296, 298, 285, 290, 300, 300, 300, 300, 294]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316674552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203962024-02-15T03:06:38.490Z{"msg":"[2]R generation[0..12]: [181, 182, 175, 179, 180, 173, 177, 182, 182, 182, 182, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.31669148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203972024-02-15T03:06:38.491Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316718287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203982024-02-15T03:06:38.491Z{"msg":"Max found gen is 183","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316733395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
203992024-02-15T03:06:38.491Z{"msg":"Generation requested: 184 >= found:183","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316748323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204002024-02-15T03:06:38.491Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.31676814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204012024-02-15T03:06:38.491Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316781849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"":"downstairs","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204022024-02-15T03:06:38.491Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316799236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204032024-02-15T03:06:38.491Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316826643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"0","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204042024-02-15T03:06:38.491Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316840701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"1","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204052024-02-15T03:06:38.491Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316860459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"client":"2","":"downstairs","session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204062024-02-15T03:06:38.491Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316874497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204072024-02-15T03:06:38.491Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316893615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204082024-02-15T03:06:38.491Z{"msg":"a54802b6-7a73-433c-a89a-db1a3f4a97cc is now active with session: abb4ed18-396b-45f6-a826-22a2d6ace6f5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316906943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204092024-02-15T03:06:38.492Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316925381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880,"session_id":"abb4ed18-396b-45f6-a826-22a2d6ace6f5"}
204102024-02-15T03:06:38.492Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:37.316938259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880}
204112024-02-15T03:06:38.492ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204122024-02-15T03:06:38.492ZLoading write count information from file "/tmp/test_repair_verify.data"
204132024-02-15T03:06:38.492ZRun Repair workload
204142024-02-15T03:06:38.492Z01/30 Write block 59 len 36864 data: 13 14 14 14 16 18 16 14 15
204152024-02-15T03:06:38.492Z02/30 Read block 248 len 24576
204162024-02-15T03:06:38.492Z03/30 Read block 170 len 4096
204172024-02-15T03:06:38.492Z04/30 Flush
204182024-02-15T03:06:38.492Z05/30 Write block 551 len 8192 data: 9 8
204192024-02-15T03:06:38.493Z06/30 Read block 431 len 40960
204202024-02-15T03:06:38.493Z07/30 Read block 257 len 8192
204212024-02-15T03:06:38.493Z08/30 Write block 304 len 28672 data: 15 18 15 14 14 16 17
204222024-02-15T03:06:38.493Z09/30 Write block 234 len 40960 data: 16 16 17 16 15 15 15 16 19 15
204232024-02-15T03:06:38.493Z10/30 Write block 314 len 20480 data: 13 12 13 13 12
204242024-02-15T03:06:38.493Z11/30 Read block 383 len 36864
204252024-02-15T03:06:38.493Z12/30 Read block 112 len 28672
204262024-02-15T03:06:38.493Z13/30 Read block 590 len 8192
204272024-02-15T03:06:38.493Z14/30 Read block 510 len 12288
204282024-02-15T03:06:38.493Z15/30 Flush
204292024-02-15T03:06:38.493Z16/30 Write block 19 len 24576 data: 10 8 7 7 6 9
204302024-02-15T03:06:38.493Z17/30 Flush
204312024-02-15T03:06:38.516Z18/30 Write block 377 len 20480 data: 16 16 14 13 14
204322024-02-15T03:06:38.516Z19/30 Write block 299 len 12288 data: 15 15 14
204332024-02-15T03:06:38.516Z20/30 Read block 308 len 24576
204342024-02-15T03:06:38.516Z21/30 Read block 348 len 20480
204352024-02-15T03:06:38.516Z22/30 Read block 174 len 28672
204362024-02-15T03:06:38.516Z23/30 Write block 133 len 40960 data: 14 15 14 13 14 14 13 15 17 15
204372024-02-15T03:06:38.516Z24/30 Read block 231 len 40960
204382024-02-15T03:06:38.516Z25/30 Read block 96 len 16384
204392024-02-15T03:06:38.516Z26/30 Read block 471 len 16384
204402024-02-15T03:06:38.517Z27/30 Read block 355 len 36864
204412024-02-15T03:06:38.517Z28/30 Read block 415 len 12288
204422024-02-15T03:06:38.517Z29/30 Read block 327 len 20480
204432024-02-15T03:06:38.517Z30/30 Write block 85 len 4096 data: 11
204442024-02-15T03:06:38.517Z----------------------------------------------------------------
204452024-02-15T03:06:38.517Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:26
204462024-02-15T03:06:38.517ZStates: Active Active Active
204472024-02-15T03:06:38.517ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204482024-02-15T03:06:38.517Z 5 Acked 1004 Write 2 Done Done Done false
204492024-02-15T03:06:38.517Z 6 Acked 1005 Read 10 Done Sent Done false
204502024-02-15T03:06:38.517Z 7 Acked 1006 Read 2 Done Sent Done false
204512024-02-15T03:06:38.517Z 8 Acked 1007 Write 7 Done Sent Done false
204522024-02-15T03:06:38.517Z 9 Acked 1008 Write 10 Done Sent Done false
204532024-02-15T03:06:38.517Z 10 Acked 1009 Write 5 Done Sent Done false
204542024-02-15T03:06:38.518Z 11 Acked 1010 Read 9 Done Sent Done false
204552024-02-15T03:06:38.518Z 12 Acked 1011 Read 7 Done Sent Done false
204562024-02-15T03:06:38.518Z 13 Acked 1012 Read 2 Done Sent Done false
204572024-02-15T03:06:38.518Z 14 Acked 1013 Read 3 Done Sent Done false
204582024-02-15T03:06:38.518Z 15 Acked 1014 Flush 0 Done Sent Done false
204592024-02-15T03:06:38.518Z 16 Acked 1015 Write 6 Done Sent Done false
204602024-02-15T03:06:38.518Z 17 Acked 1016 Flush 0 Done Sent Done false
204612024-02-15T03:06:38.518Z 18 Acked 1017 Write 5 Done Sent Done false
204622024-02-15T03:06:38.518Z 19 Acked 1018 Write 3 Done Sent Done false
204632024-02-15T03:06:38.518Z 20 Acked 1019 Read 6 Done Sent Done false
204642024-02-15T03:06:38.518Z 21 Acked 1020 Read 5 Done Sent Done false
204652024-02-15T03:06:38.518Z 22 Acked 1021 Read 7 Done Sent Done false
204662024-02-15T03:06:38.518Z 23 Acked 1022 Write 10 Done Sent Done false
204672024-02-15T03:06:38.518Z 24 Acked 1023 Read 10 Done Sent Done false
204682024-02-15T03:06:38.518Z 25 Acked 1024 Read 4 Done Sent Done false
204692024-02-15T03:06:38.518Z 26 Acked 1025 Read 4 Done Sent Done false
204702024-02-15T03:06:38.518Z 27 Acked 1026 Read 9 Done Sent Done false
204712024-02-15T03:06:38.518Z 28 Acked 1027 Read 3 Done Sent Done false
204722024-02-15T03:06:38.518Z 29 Acked 1028 Read 5 Sent Sent Done false
204732024-02-15T03:06:38.518Z 30 Acked 1029 Write 1 Sent Sent Sent false
204742024-02-15T03:06:38.518Z STATES DS:0 DS:1 DS:2 TOTAL
204752024-02-15T03:06:38.518Z New 0 0 0 0
204762024-02-15T03:06:38.518Z Sent 2 25 1 28
204772024-02-15T03:06:38.518Z Done 24 1 25 50
204782024-02-15T03:06:38.518Z Skipped 0 0 0 0
204792024-02-15T03:06:38.518Z Error 0 0 0 0
204802024-02-15T03:06:38.519ZLast Flush: 1016 1003 1016
204812024-02-15T03:06:38.519ZDownstairs last five completed: 1003 1002 1001 1000
204822024-02-15T03:06:38.519ZUpstairs last five completed: 30 29 28 27 26
204832024-02-15T03:06:38.519Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:37.349752267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1880}
204842024-02-15T03:06:38.519ZWrote out file "/tmp/test_repair_verify.data"
204852024-02-15T03:06:38.519Z
204862024-02-15T03:06:38.519ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204872024-02-15T03:06:38.519ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204882024-02-15T03:06:38.519Z 0 000-019 184 181 184 303 299 303 F F F <---
204892024-02-15T03:06:38.519Z 1 020-039 184 182 184 303 300 303 F F F <---
204902024-02-15T03:06:38.519Z 2 040-059 184 184 184 301 301 301 F F F
204912024-02-15T03:06:38.519Z 3 060-079 184 184 184 301 301 301 F F F
204922024-02-15T03:06:38.519Z 4 080-099 180 180 180 298 298 298 T F T <---
204932024-02-15T03:06:38.519Z 5 100-119 173 173 173 285 285 285 F F F
204942024-02-15T03:06:38.520Z 6 120-139 177 177 177 290 290 290 T F T <---
204952024-02-15T03:06:38.520Z 7 140-159 182 182 182 300 300 300 T F T <---
204962024-02-15T03:06:38.520Z 8 160-179 182 182 182 300 300 300 F F F
204972024-02-15T03:06:38.520Z 9 180-199 182 182 182 300 300 300 F F F
204982024-02-15T03:06:38.520Z 10 200-219 182 182 182 300 300 300 F F F
204992024-02-15T03:06:38.520Z 11 220-239 184 178 184 302 294 302 F F F <---
205002024-02-15T03:06:38.520Z 12 240-259 184 182 184 302 300 302 F F F <---
205012024-02-15T03:06:38.520Z 13 260-279 182 182 182 300 300 300 F F F
205022024-02-15T03:06:38.520Z 14 280-299 182 182 182 300 300 300 T F T <---
205032024-02-15T03:06:38.520Z 15 300-319 184 182 184 302 300 302 T F T <---
205042024-02-15T03:06:38.520Z 16 320-339 181 181 181 299 299 299 F F F
205052024-02-15T03:06:38.520Z 17 340-359 180 180 180 298 298 298 F F F
205062024-02-15T03:06:38.520Z 18 360-379 182 182 182 300 300 300 T F T <---
205072024-02-15T03:06:38.520Z 19 380-399 182 182 182 300 300 300 T F T <---
205082024-02-15T03:06:38.520Z 20 400-419 180 180 180 298 298 298 F F F
205092024-02-15T03:06:38.520Z 21 420-439 177 177 177 290 290 290 F F F
205102024-02-15T03:06:38.520Z 22 440-459 177 177 177 290 290 290 F F F
205112024-02-15T03:06:38.520Z 23 460-479 178 178 178 293 293 293 F F F
205122024-02-15T03:06:38.520Z 24 480-499 182 182 182 300 300 300 F F F
205132024-02-15T03:06:38.520Z 25 500-519 182 182 182 300 300 300 F F F
205142024-02-15T03:06:38.520Z 26 520-539 180 180 180 297 297 297 F F F
205152024-02-15T03:06:38.520Z 27 540-559 184 177 184 302 290 302 F T F <---
205162024-02-15T03:06:38.521Z 28 560-579 182 182 182 300 300 300 F F F
205172024-02-15T03:06:38.521Z 29 580-599 182 182 182 300 300 300 F F F
205182024-02-15T03:06:38.521ZMax gen: 184, Max flush: 303
205192024-02-15T03:06:38.521ZError: Difference in extent metadata found!
205202024-02-15T03:06:38.521ZA Difference in extent metadata is expected here
205212024-02-15T03:06:38.521ZOn loop 92
205222024-02-15T03:06:38.521Z
205232024-02-15T03:06:38.540ZVerifying data now
205242024-02-15T03:06:44.120ZNo change after verify
205252024-02-15T03:06:44.121ZLoop: 92 Downstairs dump after verify (and repair):
205262024-02-15T03:06:44.140ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205272024-02-15T03:06:44.141Z 0 000-019 184 184 184 303 303 303 F F F
205282024-02-15T03:06:44.141Z 1 020-039 184 184 184 303 303 303 F F F
205292024-02-15T03:06:44.141Z 2 040-059 184 184 184 301 301 301 F F F
205302024-02-15T03:06:44.141Z 3 060-079 184 184 184 301 301 301 F F F
205312024-02-15T03:06:44.142Z 4 080-099 185 185 185 304 304 304 F F F
205322024-02-15T03:06:44.142Z 5 100-119 173 173 173 285 285 285 F F F
205332024-02-15T03:06:44.142Z 6 120-139 185 185 185 304 304 304 F F F
205342024-02-15T03:06:44.142Z 7 140-159 185 185 185 304 304 304 F F F
205352024-02-15T03:06:44.142Z 8 160-179 182 182 182 300 300 300 F F F
205362024-02-15T03:06:44.142Z 9 180-199 182 182 182 300 300 300 F F F
205372024-02-15T03:06:44.142Z 10 200-219 182 182 182 300 300 300 F F F
205382024-02-15T03:06:44.143Z 11 220-239 184 184 184 302 302 302 F F F
205392024-02-15T03:06:44.143Z 12 240-259 184 184 184 302 302 302 F F F
205402024-02-15T03:06:44.143Z 13 260-279 182 182 182 300 300 300 F F F
205412024-02-15T03:06:44.144Z 14 280-299 185 185 185 304 304 304 F F F
205422024-02-15T03:06:44.144Z 15 300-319 185 185 185 304 304 304 F F F
205432024-02-15T03:06:44.144Z 16 320-339 181 181 181 299 299 299 F F F
205442024-02-15T03:06:44.167Z 17 340-359 180 180 180 298 298 298 F F F
205452024-02-15T03:06:44.167Z 18 360-379 185 185 185 304 304 304 F F F
205462024-02-15T03:06:44.167Z 19 380-399 185 185 185 304 304 304 F F F
205472024-02-15T03:06:44.167Z 20 400-419 180 180 180 298 298 298 F F F
205482024-02-15T03:06:44.167Z 21 420-439 177 177 177 290 290 290 F F F
205492024-02-15T03:06:44.167Z 22 440-459 177 177 177 290 290 290 F F F
205502024-02-15T03:06:44.167Z 23 460-479 178 178 178 293 293 293 F F F
205512024-02-15T03:06:44.168Z 24 480-499 182 182 182 300 300 300 F F F
205522024-02-15T03:06:44.168Z 25 500-519 182 182 182 300 300 300 F F F
205532024-02-15T03:06:44.168Z 26 520-539 180 180 180 297 297 297 F F F
205542024-02-15T03:06:44.168Z 27 540-559 184 184 184 302 302 302 F F F
205552024-02-15T03:06:44.168Z 28 560-579 182 182 182 300 300 300 F F F
205562024-02-15T03:06:44.168Z 29 580-599 182 182 182 300 300 300 F F F
205572024-02-15T03:06:44.168ZMax gen: 185, Max flush: 304
205582024-02-15T03:06:44.168Z
205592024-02-15T03:06:44.168ZBegin loop 93
205602024-02-15T03:06:44.168ZDownstairs to restart: 0
205612024-02-15T03:06:44.168Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.038809776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888}
205622024-02-15T03:06:44.168Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039059445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888}
205632024-02-15T03:06:44.168Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039077832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888}
205642024-02-15T03:06:44.169Z{"msg":"Crucible cf1fad8d-55dc-490a-b113-6c0482361fa3 has session id: 3127be98-5c71-4656-96ec-1f402b62989a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039091261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205652024-02-15T03:06:44.169Z{"msg":"Upstairs opts: Upstairs UUID: cf1fad8d-55dc-490a-b113-6c0482361fa3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039125426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205662024-02-15T03:06:44.169Z{"msg":"Crucible stats registered with UUID: cf1fad8d-55dc-490a-b113-6c0482361fa3","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039151143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205672024-02-15T03:06:44.169ZCrucible runtime is spawned
205682024-02-15T03:06:44.169Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039267698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888}
205692024-02-15T03:06:44.169Z{"msg":"cf1fad8d-55dc-490a-b113-6c0482361fa3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039367196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205702024-02-15T03:06:44.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039393373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205712024-02-15T03:06:44.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039409261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205722024-02-15T03:06:44.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039435587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205732024-02-15T03:06:44.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039471663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205742024-02-15T03:06:44.169Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039509928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205752024-02-15T03:06:44.169Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039545613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205762024-02-15T03:06:44.169Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039578909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205772024-02-15T03:06:44.170Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039610785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205782024-02-15T03:06:44.170Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039647181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205792024-02-15T03:06:44.170Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:43.03973431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205802024-02-15T03:06:44.170Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:43.039789383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205812024-02-15T03:06:44.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039821199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205822024-02-15T03:06:44.170Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:43.039851525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205832024-02-15T03:06:44.170Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039876032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205842024-02-15T03:06:44.170Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.039901029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205852024-02-15T03:06:44.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.03997653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205862024-02-15T03:06:44.170Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.040024293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205872024-02-15T03:06:44.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.040470308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205882024-02-15T03:06:44.170Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.04053224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205892024-02-15T03:06:44.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.040575075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205902024-02-15T03:06:44.171Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.04061593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205912024-02-15T03:06:44.171Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041138364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205922024-02-15T03:06:44.171Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041281266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205932024-02-15T03:06:44.171Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041668867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205942024-02-15T03:06:44.171Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041758536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205952024-02-15T03:06:44.171Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041820919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205962024-02-15T03:06:44.171Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041841696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205972024-02-15T03:06:44.171Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041856924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205982024-02-15T03:06:44.171Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:43.041871282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
205992024-02-15T03:06:45.171Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.041458497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206002024-02-15T03:06:45.171Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.041535427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206012024-02-15T03:06:45.185Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.042445863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206022024-02-15T03:06:45.185Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.042491838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206032024-02-15T03:06:45.185Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.042996554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206042024-02-15T03:06:45.185Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043610937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206052024-02-15T03:06:45.185Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043674999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206062024-02-15T03:06:45.185Z{"msg":"[0]R flush_numbers[0..12]: [303, 303, 301, 301, 304, 285, 304, 304, 300, 300, 300, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043723233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206072024-02-15T03:06:45.185Z{"msg":"[0]R generation[0..12]: [184, 184, 184, 184, 185, 173, 185, 185, 182, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043754589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206082024-02-15T03:06:45.185Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043774177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206092024-02-15T03:06:45.186Z{"msg":"[1]R flush_numbers[0..12]: [303, 303, 301, 301, 304, 285, 304, 304, 300, 300, 300, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043800634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206102024-02-15T03:06:45.186Z{"msg":"[1]R generation[0..12]: [184, 184, 184, 184, 185, 173, 185, 185, 182, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043835969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206112024-02-15T03:06:45.186Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043854427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206122024-02-15T03:06:45.186Z{"msg":"[2]R flush_numbers[0..12]: [303, 303, 301, 301, 304, 285, 304, 304, 300, 300, 300, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043880324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206132024-02-15T03:06:45.186ZWait for a query_work_queue command to finish before sending IO
206142024-02-15T03:06:45.186Z{"msg":"[2]R generation[0..12]: [184, 184, 184, 184, 185, 173, 185, 185, 182, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.0439101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206152024-02-15T03:06:45.186Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043937446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206162024-02-15T03:06:45.186Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.043955884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206172024-02-15T03:06:45.186Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.04399177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206182024-02-15T03:06:45.187Z{"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044022166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206192024-02-15T03:06:45.187Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044040853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"":"downstairs","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206202024-02-15T03:06:45.187Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044059101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206212024-02-15T03:06:45.187Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044074529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"0","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206222024-02-15T03:06:45.187Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044106255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"1","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206232024-02-15T03:06:45.187Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044123443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"client":"2","":"downstairs","session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206242024-02-15T03:06:45.187Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.04414868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206252024-02-15T03:06:45.187Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044165098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206262024-02-15T03:06:45.188Z{"msg":"cf1fad8d-55dc-490a-b113-6c0482361fa3 is now active with session: 3127be98-5c71-4656-96ec-1f402b62989a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044203733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206272024-02-15T03:06:45.188Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044220871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888,"session_id":"3127be98-5c71-4656-96ec-1f402b62989a"}
206282024-02-15T03:06:45.188Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:44.044245868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888}
206292024-02-15T03:06:45.188ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206302024-02-15T03:06:45.188ZLoading write count information from file "/tmp/test_repair_verify.data"
206312024-02-15T03:06:45.188ZRun Repair workload
206322024-02-15T03:06:45.188Z01/30 Write block 302 len 36864 data: 13 15 16 19 16 15 15 17 18
206332024-02-15T03:06:45.188Z02/30 Write block 319 len 4096 data: 11
206342024-02-15T03:06:45.188Z03/30 Read block 231 len 20480
206352024-02-15T03:06:45.189Z04/30 Write block 189 len 4096 data: 21
206362024-02-15T03:06:45.189Z05/30 Flush
206372024-02-15T03:06:45.189Z06/30 Flush
206382024-02-15T03:06:45.189Z07/30 Write block 385 len 40960 data: 15 14 15 16 18 16 15 13 12 13
206392024-02-15T03:06:45.189Z08/30 Read block 24 len 4096
206402024-02-15T03:06:45.189Z09/30 Read block 497 len 8192
206412024-02-15T03:06:45.189Z10/30 Read block 210 len 8192
206422024-02-15T03:06:45.189Z11/30 Write block 178 len 40960 data: 11 14 14 13 12 15 18 20 19 17
206432024-02-15T03:06:45.189Z12/30 Read block 373 len 20480
206442024-02-15T03:06:45.189Z13/30 Flush
206452024-02-15T03:06:45.198Z14/30 Write block 584 len 32768 data: 16 16 16 14 12 16 17 18
206462024-02-15T03:06:45.198Z15/30 Write block 177 len 32768 data: 11 12 15 15 14 13 16 19
206472024-02-15T03:06:45.198Z16/30 Flush
206482024-02-15T03:06:45.198Z17/30 Write block 197 len 36864 data: 15 11 13 14 11 9 11 12 14
206492024-02-15T03:06:45.198Z18/30 Read block 414 len 4096
206502024-02-15T03:06:45.198Z19/30 Read block 40 len 16384
206512024-02-15T03:06:45.198Z20/30 Write block 97 len 32768 data: 18 18 16 13 15 14 12 13
206522024-02-15T03:06:45.198Z21/30 Write block 313 len 24576 data: 14 14 13 14 14 13
206532024-02-15T03:06:45.198Z22/30 Flush
206542024-02-15T03:06:45.198Z23/30 Write block 69 len 40960 data: 14 13 14 14 8 10 13 15 13 11
206552024-02-15T03:06:45.198Z24/30 Flush
206562024-02-15T03:06:45.202Z25/30 Write block 290 len 24576 data: 23 22 21 20 20 17
206572024-02-15T03:06:45.202Z26/30 Flush
206582024-02-15T03:06:45.202Z27/30 Write block 30 len 24576 data: 14 15 14 17 17 18
206592024-02-15T03:06:45.202Z28/30 Write block 582 len 20480 data: 18 17 17 17 17
206602024-02-15T03:06:45.203Z29/30 Write block 201 len 16384 data: 12 10 12 13
206612024-02-15T03:06:45.203Z30/30 Read block 237 len 20480
206622024-02-15T03:06:45.212Z----------------------------------------------------------------
206632024-02-15T03:06:45.212Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
206642024-02-15T03:06:45.212ZStates: Active Active Active
206652024-02-15T03:06:45.212ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206662024-02-15T03:06:45.212Z 1 Acked 1000 Write 9 Done Done Done false
206672024-02-15T03:06:45.212Z 2 Acked 1001 Write 1 Done Done Done false
206682024-02-15T03:06:45.212Z 3 Acked 1002 Read 5 Done Done Done false
206692024-02-15T03:06:45.212Z 4 Acked 1003 Write 1 Sent Done Done false
206702024-02-15T03:06:45.213Z 5 Acked 1004 Flush 0 Sent Done Done false
206712024-02-15T03:06:45.213Z 6 Acked 1005 Flush 0 Sent Done Done false
206722024-02-15T03:06:45.213Z 7 Acked 1006 Write 10 Sent Done Done false
206732024-02-15T03:06:45.213Z 8 Acked 1007 Read 1 Sent Done Done false
206742024-02-15T03:06:45.213Z 9 Acked 1008 Read 2 Sent Done Done false
206752024-02-15T03:06:45.213Z 10 Acked 1009 Read 2 Sent Done Done false
206762024-02-15T03:06:45.213Z 11 Acked 1010 Write 10 Sent Done Done false
206772024-02-15T03:06:45.213Z 12 Acked 1011 Read 5 Sent Done Done false
206782024-02-15T03:06:45.213Z 13 Acked 1012 Flush 0 Sent Done Done false
206792024-02-15T03:06:45.213Z 14 Acked 1013 Write 8 Sent Done Done false
206802024-02-15T03:06:45.213Z 15 Acked 1014 Write 8 Sent Done Done false
206812024-02-15T03:06:45.213Z 16 Acked 1015 Flush 0 Sent Done Done false
206822024-02-15T03:06:45.214Z 17 Acked 1016 Write 9 Sent Done Done false
206832024-02-15T03:06:45.214Z 18 Acked 1017 Read 1 Sent Done Done false
206842024-02-15T03:06:45.214Z 19 Acked 1018 Read 4 Sent Done Done false
206852024-02-15T03:06:45.214Z 20 Acked 1019 Write 8 Sent Done Done false
206862024-02-15T03:06:45.214Z 21 Acked 1020 Write 6 Sent Done Done false
206872024-02-15T03:06:45.214Z 22 Acked 1021 Flush 0 Sent Done Done false
206882024-02-15T03:06:45.214Z 23 Acked 1022 Write 10 Sent Done Done false
206892024-02-15T03:06:45.214Z 24 Acked 1023 Flush 0 Sent Done Done false
206902024-02-15T03:06:45.214Z 25 Acked 1024 Write 6 Sent Done Done false
206912024-02-15T03:06:45.214Z 26 Acked 1025 Flush 0 Sent Done Done false
206922024-02-15T03:06:45.214Z 27 Acked 1026 Write 6 Sent Done Done false
206932024-02-15T03:06:45.214Z 28 Acked 1027 Write 5 Sent Done Done false
206942024-02-15T03:06:45.214Z 29 Acked 1028 Write 4 Sent Done Done false
206952024-02-15T03:06:45.215Z 30 Acked 1029 Read 5 Sent Done Sent false
206962024-02-15T03:06:45.215Z STATES DS:0 DS:1 DS:2 TOTAL
206972024-02-15T03:06:45.215Z New 0 0 0 0
206982024-02-15T03:06:45.215Z Sent 27 0 1 28
206992024-02-15T03:06:45.215Z Done 3 30 29 62
207002024-02-15T03:06:45.215Z Skipped 0 0 0 0
207012024-02-15T03:06:45.215Z Error 0 0 0 0
207022024-02-15T03:06:45.215ZLast Flush: 0 1025 1025
207032024-02-15T03:06:45.215ZDownstairs last five completed:
207042024-02-15T03:06:45.215ZUpstairs last five completed: 30 29 28 27 26
207052024-02-15T03:06:45.215Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:44.081529976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1888}
207062024-02-15T03:06:45.215ZWrote out file "/tmp/test_repair_verify.data"
207072024-02-15T03:06:45.215Z
207082024-02-15T03:06:45.221ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207092024-02-15T03:06:45.232ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207102024-02-15T03:06:45.232Z 0 000-019 184 184 184 303 303 303 F F F
207112024-02-15T03:06:45.232Z 1 020-039 184 184 184 303 303 303 F T T <---
207122024-02-15T03:06:45.232Z 2 040-059 184 184 184 301 301 301 F F F
207132024-02-15T03:06:45.232Z 3 060-079 184 186 186 301 310 310 F F F <---
207142024-02-15T03:06:45.232Z 4 080-099 185 186 186 304 309 309 F F F <---
207152024-02-15T03:06:45.232Z 5 100-119 173 186 186 285 309 309 F F F <---
207162024-02-15T03:06:45.233Z 6 120-139 185 185 185 304 304 304 F F F
207172024-02-15T03:06:45.233Z 7 140-159 185 185 185 304 304 304 F F F
207182024-02-15T03:06:45.238Z 8 160-179 182 186 186 300 308 308 F F F <---
207192024-02-15T03:06:45.238Z 9 180-199 182 186 186 300 309 309 F F F <---
207202024-02-15T03:06:45.238Z 10 200-219 182 186 186 300 309 309 F T T <---
207212024-02-15T03:06:45.238Z 11 220-239 184 184 184 302 302 302 F F F
207222024-02-15T03:06:45.238Z 12 240-259 184 184 184 302 302 302 F F F
207232024-02-15T03:06:45.238Z 13 260-279 182 182 182 300 300 300 F F F
207242024-02-15T03:06:45.239Z 14 280-299 185 186 186 304 311 311 F F F <---
207252024-02-15T03:06:45.239Z 15 300-319 185 186 186 304 309 309 T F F <---
207262024-02-15T03:06:45.239Z 16 320-339 181 181 181 299 299 299 F F F
207272024-02-15T03:06:45.239Z 17 340-359 180 180 180 298 298 298 F F F
207282024-02-15T03:06:45.239Z 18 360-379 185 185 185 304 304 304 F F F
207292024-02-15T03:06:45.239Z 19 380-399 185 186 186 304 307 307 F F F <---
207302024-02-15T03:06:45.239Z 20 400-419 180 180 180 298 298 298 F F F
207312024-02-15T03:06:45.239Z 21 420-439 177 177 177 290 290 290 F F F
207322024-02-15T03:06:45.239Z 22 440-459 177 177 177 290 290 290 F F F
207332024-02-15T03:06:45.239Z 23 460-479 178 178 178 293 293 293 F F F
207342024-02-15T03:06:45.239Z 24 480-499 182 182 182 300 300 300 F F F
207352024-02-15T03:06:45.239Z 25 500-519 182 182 182 300 300 300 F F F
207362024-02-15T03:06:45.240Z 26 520-539 180 180 180 297 297 297 F F F
207372024-02-15T03:06:45.240Z 27 540-559 184 184 184 302 302 302 F F F
207382024-02-15T03:06:45.240Z 28 560-579 182 182 182 300 300 300 F F F
207392024-02-15T03:06:45.240Z 29 580-599 182 186 186 300 308 308 F T T <---
207402024-02-15T03:06:45.240ZMax gen: 186, Max flush: 311
207412024-02-15T03:06:45.240ZError: Difference in extent metadata found!
207422024-02-15T03:06:45.240ZA Difference in extent metadata is expected here
207432024-02-15T03:06:45.240ZOn loop 93
207442024-02-15T03:06:45.240Z
207452024-02-15T03:06:45.245ZVerifying data now
207462024-02-15T03:06:50.822ZNo change after verify
207472024-02-15T03:06:50.822ZLoop: 93 Downstairs dump after verify (and repair):
207482024-02-15T03:06:50.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207492024-02-15T03:06:50.843Z 0 000-019 184 184 184 303 303 303 F F F
207502024-02-15T03:06:50.843Z 1 020-039 187 187 187 312 312 312 F F F
207512024-02-15T03:06:50.844Z 2 040-059 184 184 184 301 301 301 F F F
207522024-02-15T03:06:50.844Z 3 060-079 186 186 186 310 310 310 F F F
207532024-02-15T03:06:50.844Z 4 080-099 186 186 186 309 309 309 F F F
207542024-02-15T03:06:50.844Z 5 100-119 186 186 186 309 309 309 F F F
207552024-02-15T03:06:50.844Z 6 120-139 185 185 185 304 304 304 F F F
207562024-02-15T03:06:50.844Z 7 140-159 185 185 185 304 304 304 F F F
207572024-02-15T03:06:50.844Z 8 160-179 186 186 186 308 308 308 F F F
207582024-02-15T03:06:50.845Z 9 180-199 186 186 186 309 309 309 F F F
207592024-02-15T03:06:50.845Z 10 200-219 187 187 187 312 312 312 F F F
207602024-02-15T03:06:50.845Z 11 220-239 184 184 184 302 302 302 F F F
207612024-02-15T03:06:50.845Z 12 240-259 184 184 184 302 302 302 F F F
207622024-02-15T03:06:50.845Z 13 260-279 182 182 182 300 300 300 F F F
207632024-02-15T03:06:50.846Z 14 280-299 186 186 186 311 311 311 F F F
207642024-02-15T03:06:50.846Z 15 300-319 186 186 186 309 309 309 F F F
207652024-02-15T03:06:50.846Z 16 320-339 181 181 181 299 299 299 F F F
207662024-02-15T03:06:50.846Z 17 340-359 180 180 180 298 298 298 F F F
207672024-02-15T03:06:50.846Z 18 360-379 185 185 185 304 304 304 F F F
207682024-02-15T03:06:50.846Z 19 380-399 186 186 186 307 307 307 F F F
207692024-02-15T03:06:50.847Z 20 400-419 180 180 180 298 298 298 F F F
207702024-02-15T03:06:50.847Z 21 420-439 177 177 177 290 290 290 F F F
207712024-02-15T03:06:50.847Z 22 440-459 177 177 177 290 290 290 F F F
207722024-02-15T03:06:50.847Z 23 460-479 178 178 178 293 293 293 F F F
207732024-02-15T03:06:50.847Z 24 480-499 182 182 182 300 300 300 F F F
207742024-02-15T03:06:50.848Z 25 500-519 182 182 182 300 300 300 F F F
207752024-02-15T03:06:50.848Z 26 520-539 180 180 180 297 297 297 F F F
207762024-02-15T03:06:50.848Z 27 540-559 184 184 184 302 302 302 F F F
207772024-02-15T03:06:50.848Z 28 560-579 182 182 182 300 300 300 F F F
207782024-02-15T03:06:50.849Z 29 580-599 187 187 187 312 312 312 F F F
207792024-02-15T03:06:50.849ZMax gen: 187, Max flush: 312
207802024-02-15T03:06:50.866Z
207812024-02-15T03:06:50.867ZBegin loop 94
207822024-02-15T03:06:50.867ZDownstairs to restart: 1
207832024-02-15T03:06:50.867Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.731481454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896}
207842024-02-15T03:06:50.867Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
207852024-02-15T03:06:50.867Z,"time":"2024-02-15T03:06:49.731824581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896}
207862024-02-15T03:06:50.868Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.731923019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896}
207872024-02-15T03:06:50.868Z{"msg":"Crucible cde09775-9564-46f2-8602-9b0f2cf4d543 has session id: 9464239d-b1ba-48dd-be34-f1360b3d1cb9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.731961174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207882024-02-15T03:06:50.868Z{"msg":"Upstairs opts: Upstairs UUID: cde09775-9564-46f2-8602-9b0f2cf4d543, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732007438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207892024-02-15T03:06:50.868Z{"msg":"Crucible stats registered with UUID: cde09775-9564-46f2-8602-9b0f2cf4d543","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732027445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207902024-02-15T03:06:50.869Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732052452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896}
207912024-02-15T03:06:50.869Z{"msg":"cde09775-9564-46f2-8602-9b0f2cf4d543 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.73206778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207922024-02-15T03:06:50.869Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732099456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207932024-02-15T03:06:50.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732118994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207942024-02-15T03:06:50.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732137792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207952024-02-15T03:06:50.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732181926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207962024-02-15T03:06:50.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.73223533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207972024-02-15T03:06:50.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732291752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207982024-02-15T03:06:50.871Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732329688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
207992024-02-15T03:06:50.871Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732382411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208002024-02-15T03:06:50.871Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732416627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208012024-02-15T03:06:50.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732527173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208022024-02-15T03:06:50.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732578116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208032024-02-15T03:06:50.871Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:49.732611512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208042024-02-15T03:06:50.871Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:49.732640929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208052024-02-15T03:06:50.872Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:49.732667495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208062024-02-15T03:06:50.872Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732691182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208072024-02-15T03:06:50.872Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732715009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208082024-02-15T03:06:50.872Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.732916304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208092024-02-15T03:06:50.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.733369097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208102024-02-15T03:06:50.872Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.733408392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208112024-02-15T03:06:50.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.733444188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208122024-02-15T03:06:50.872Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.733482433Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208132024-02-15T03:06:50.872Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734023795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208142024-02-15T03:06:50.872Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734060151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208152024-02-15T03:06:50.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734681843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208162024-02-15T03:06:50.872Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734720498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208172024-02-15T03:06:50.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734736706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208182024-02-15T03:06:50.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734754744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208192024-02-15T03:06:50.873Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734779001Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208202024-02-15T03:06:50.873Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:49.734793079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208212024-02-15T03:06:52.271Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.734393232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208222024-02-15T03:06:52.279Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.734540314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208232024-02-15T03:06:52.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.735407155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208242024-02-15T03:06:52.279Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.73544555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208252024-02-15T03:06:52.279Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.736061123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208262024-02-15T03:06:52.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.73672165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208272024-02-15T03:06:52.279Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.736762805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208282024-02-15T03:06:52.280Z{"msg":"[0]R flush_numbers[0..12]: [303, 312, 301, 310, 309, 309, 304, 304, 308, 309, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.736797091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208292024-02-15T03:06:52.280Z{"msg":"[0]R generation[0..12]: [184, 187, 184, 186, 186, 186, 185, 185, 186, 186, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.736839665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208302024-02-15T03:06:52.280Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.73688442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208312024-02-15T03:06:52.280Z{"msg":"[1]R flush_numbers[0..12]: [303, 312, 301, 310, 309, 309, 304, 304, 308, 309, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.736929624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208322024-02-15T03:06:52.280Z{"msg":"[1]R generation[0..12]: [184, 187, 184, 186, 186, 186, 185, 185, 186, 186, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.736973749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208332024-02-15T03:06:52.280Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737018293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208342024-02-15T03:06:52.280Z{"msg":"[2]R flush_numbers[0..12]: [303, 312, 301, 310, 309, 309, 304, 304, 308, 309, 312, 302]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
208352024-02-15T03:06:52.280Z,"time":"2024-02-15T03:06:50.737065367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208362024-02-15T03:06:52.280Z{"msg":"[2]R generation[0..12]: [184, 187, 184, 186, 186, 186, 185, 185, 186, 186, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737119401Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208372024-02-15T03:06:52.280Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737151637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208382024-02-15T03:06:52.280Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737190312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208392024-02-15T03:06:52.280Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737219838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208402024-02-15T03:06:52.280Z{"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737236996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208412024-02-15T03:06:52.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737262392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"":"downstairs","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208422024-02-15T03:06:52.281Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.73728024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208432024-02-15T03:06:52.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737303467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"0","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208442024-02-15T03:06:52.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737321225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"1","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208452024-02-15T03:06:52.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737345732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"client":"2","":"downstairs","session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208462024-02-15T03:06:52.281Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.73736351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208472024-02-15T03:06:52.281Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737402095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208482024-02-15T03:06:52.281Z{"msg":"cde09775-9564-46f2-8602-9b0f2cf4d543 is now active with session: 9464239d-b1ba-48dd-be34-f1360b3d1cb9","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737419873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208492024-02-15T03:06:52.281Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737447069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896,"session_id":"9464239d-b1ba-48dd-be34-f1360b3d1cb9"}
208502024-02-15T03:06:52.281Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:50.737470977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896}
208512024-02-15T03:06:52.281ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208522024-02-15T03:06:52.281ZLoading write count information from file "/tmp/test_repair_verify.data"
208532024-02-15T03:06:52.282ZRun Repair workload
208542024-02-15T03:06:52.282Z01/30 Write block 527 len 8192 data: 11 13
208552024-02-15T03:06:52.282Z02/30 Write block 351 len 8192 data: 20 19
208562024-02-15T03:06:52.282Z03/30 Read block 332 len 8192
208572024-02-15T03:06:52.282Z04/30 Write block 373 len 28672 data: 14 13 13 14 17 17 15
208582024-02-15T03:06:52.282Z05/30 Flush
208592024-02-15T03:06:52.282Z06/30 Write block 85 len 32768 data: 12 14 14 11 10 11 12 10
208602024-02-15T03:06:52.282Z07/30 Write block 1 len 40960 data: 4 5 9 12 12 13 12 14 12 17
208612024-02-15T03:06:52.282Z08/30 Read block 521 len 32768
208622024-02-15T03:06:52.282Z09/30 Write block 241 len 8192 data: 17 20
208632024-02-15T03:06:52.282Z10/30 Read block 169 len 4096
208642024-02-15T03:06:52.282Z11/30 Write block 136 len 8192 data: 14 15
208652024-02-15T03:06:52.282Z12/30 Read block 585 len 4096
208662024-02-15T03:06:52.283Z13/30 Flush
208672024-02-15T03:06:52.283Z14/30 Write block 249 len 36864 data: 13 13 14 16 15 16 14 16 18
208682024-02-15T03:06:52.283Z15/30 Read block 235 len 24576
208692024-02-15T03:06:52.283Z16/30 Flush
208702024-02-15T03:06:52.283Z17/30 Write block 249 len 8192 data: 14 14
208712024-02-15T03:06:52.283Z18/30 Read block 113 len 36864
208722024-02-15T03:06:52.283Z19/30 Read block 448 len 32768
208732024-02-15T03:06:52.283Z20/30 Write block 156 len 32768 data: 19 20 16 13 11 14 17 15
208742024-02-15T03:06:52.283Z21/30 Write block 505 len 24576 data: 17 17 13 13 17 17
208752024-02-15T03:06:52.283Z22/30 Flush
208762024-02-15T03:06:52.283Z23/30 Flush
208772024-02-15T03:06:52.283Z24/30 Write block 309 len 40960 data: 18 19 16 12 15 15 14 15 15 14
208782024-02-15T03:06:52.283Z25/30 Read block 322 len 28672
208792024-02-15T03:06:52.284Z26/30 Read block 194 len 24576
208802024-02-15T03:06:52.284Z27/30 Write block 134 len 20480 data: 16 15 15 16 15
208812024-02-15T03:06:52.284Z28/30 Read block 209 len 20480
208822024-02-15T03:06:52.284Z29/30 Write block 186 len 28672 data: 20 18 18 22 18 17 17
208832024-02-15T03:06:52.284Z30/30 Read block 88 len 40960
208842024-02-15T03:06:52.284Z----------------------------------------------------------------
208852024-02-15T03:06:52.284Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
208862024-02-15T03:06:52.284ZStates: Active Active Active
208872024-02-15T03:06:52.284ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208882024-02-15T03:06:52.284Z 1 Acked 1000 Write 2 Done Done Done false
208892024-02-15T03:06:52.284Z 2 Acked 1001 Write 2 Done Sent Done false
208902024-02-15T03:06:52.284Z 3 Acked 1002 Read 2 Done Sent Done false
208912024-02-15T03:06:52.284Z 4 Acked 1003 Write 7 Done Sent Done false
208922024-02-15T03:06:52.285Z 5 Acked 1004 Flush 0 Done Sent Done false
208932024-02-15T03:06:52.285Z 6 Acked 1005 Write 8 Done Sent Done false
208942024-02-15T03:06:52.285Z 7 Acked 1006 Write 10 Done Sent Done false
208952024-02-15T03:06:52.285Z 8 Acked 1007 Read 8 Done Sent Done false
208962024-02-15T03:06:52.285Z 9 Acked 1008 Write 2 Done Sent Done false
208972024-02-15T03:06:52.285Z 10 Acked 1009 Read 1 Done Sent Done false
208982024-02-15T03:06:52.285Z 11 Acked 1010 Write 2 Done Sent Done false
208992024-02-15T03:06:52.285Z 12 Acked 1011 Read 1 Done Sent Done false
209002024-02-15T03:06:52.285Z 13 Acked 1012 Flush 0 Done Sent Done false
209012024-02-15T03:06:52.285Z 14 Acked 1013 Write 9 Done Sent Done false
209022024-02-15T03:06:52.285Z 15 Acked 1014 Read 6 Done Sent Done false
209032024-02-15T03:06:52.285Z 16 Acked 1015 Flush 0 Done Sent Done false
209042024-02-15T03:06:52.285Z 17 Acked 1016 Write 2 Done Sent Done false
209052024-02-15T03:06:52.286Z 18 Acked 1017 Read 9 Done Sent Done false
209062024-02-15T03:06:52.286Z 19 Acked 1018 Read 8 Done Sent Done false
209072024-02-15T03:06:52.286Z 20 Acked 1019 Write 8 Done Sent Done false
209082024-02-15T03:06:52.286Z 21 Acked 1020 Write 6 Done Sent Done false
209092024-02-15T03:06:52.286Z 22 Acked 1021 Flush 0 Done Sent Done false
209102024-02-15T03:06:52.286Z 23 Acked 1022 Flush 0 Done Sent Done false
209112024-02-15T03:06:52.286Z 24 Acked 1023 Write 10 Done Sent Done false
209122024-02-15T03:06:52.286Z 25 Acked 1024 Read 7 Done Sent Done false
209132024-02-15T03:06:52.286Z 26 Acked 1025 Read 6 Done Sent Done false
209142024-02-15T03:06:52.286Z 27 Acked 1026 Write 5 Done Sent Done false
209152024-02-15T03:06:52.286Z 28 Acked 1027 Read 5 Done Sent Done false
209162024-02-15T03:06:52.286Z 29 Acked 1028 Write 7 Done Sent Done false
209172024-02-15T03:06:52.287Z 30 Acked 1029 Read 10 Done Sent Done false
209182024-02-15T03:06:52.287Z STATES DS:0 DS:1 DS:2 TOTAL
209192024-02-15T03:06:52.287Z New 0 0 0 0
209202024-02-15T03:06:52.287Z Sent 0 29 0 29
209212024-02-15T03:06:52.287Z Done 30 1 30 61
209222024-02-15T03:06:52.292Z Skipped 0 0 0 0
209232024-02-15T03:06:52.292Z Error 0 0 0 0
209242024-02-15T03:06:52.292ZLast Flush: 1022 0 1022
209252024-02-15T03:06:52.292ZDownstairs last five completed:
209262024-02-15T03:06:52.292ZUpstairs last five completed: 30 29 28 27 26
209272024-02-15T03:06:52.292Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:50.771088434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1896}
209282024-02-15T03:06:52.292ZWrote out file "/tmp/test_repair_verify.data"
209292024-02-15T03:06:52.292Z
209302024-02-15T03:06:52.292ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209312024-02-15T03:06:52.292ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209322024-02-15T03:06:52.292Z 0 000-019 188 184 188 314 303 314 F F F <---
209332024-02-15T03:06:52.293Z 1 020-039 187 187 187 312 312 312 F F F
209342024-02-15T03:06:52.293Z 2 040-059 184 184 184 301 301 301 F F F
209352024-02-15T03:06:52.293Z 3 060-079 186 186 186 310 310 310 F F F
209362024-02-15T03:06:52.293Z 4 080-099 188 186 188 314 309 314 F F F <---
209372024-02-15T03:06:52.293Z 5 100-119 186 186 186 309 309 309 F F F
209382024-02-15T03:06:52.293Z 6 120-139 188 185 188 314 304 314 T F T <---
209392024-02-15T03:06:52.293Z 7 140-159 188 185 188 316 304 316 F F F <---
209402024-02-15T03:06:52.293Z 8 160-179 188 186 188 316 308 316 F F F <---
209412024-02-15T03:06:52.293Z 9 180-199 186 186 186 309 309 309 T F T <---
209422024-02-15T03:06:52.293Z 10 200-219 187 187 187 312 312 312 F F F
209432024-02-15T03:06:52.294Z 11 220-239 184 184 184 302 302 302 F F F
209442024-02-15T03:06:52.294Z 12 240-259 188 184 188 316 302 316 F F F <---
209452024-02-15T03:06:52.294Z 13 260-279 182 182 182 300 300 300 F F F
209462024-02-15T03:06:52.294Z 14 280-299 186 186 186 311 311 311 F F F
209472024-02-15T03:06:52.294Z 15 300-319 186 186 186 309 309 309 T F T <---
209482024-02-15T03:06:52.294Z 16 320-339 181 181 181 299 299 299 F F F
209492024-02-15T03:06:52.294Z 17 340-359 188 180 188 313 298 313 F F F <---
209502024-02-15T03:06:52.294Z 18 360-379 188 185 188 313 304 313 F F F <---
209512024-02-15T03:06:52.294Z 19 380-399 186 186 186 307 307 307 F F F
209522024-02-15T03:06:52.294Z 20 400-419 180 180 180 298 298 298 F F F
209532024-02-15T03:06:52.294Z 21 420-439 177 177 177 290 290 290 F F F
209542024-02-15T03:06:52.294Z 22 440-459 177 177 177 290 290 290 F F F
209552024-02-15T03:06:52.294Z 23 460-479 178 178 178 293 293 293 F F F
209562024-02-15T03:06:52.295Z 24 480-499 182 182 182 300 300 300 F F F
209572024-02-15T03:06:52.295Z 25 500-519 188 182 188 316 300 316 F F F <---
209582024-02-15T03:06:52.295Z 26 520-539 188 180 188 313 297 313 F T F <---
209592024-02-15T03:06:52.295Z 27 540-559 184 184 184 302 302 302 F F F
209602024-02-15T03:06:52.295Z 28 560-579 182 182 182 300 300 300 F F F
209612024-02-15T03:06:52.295Z 29 580-599 187 187 187 312 312 312 F F F
209622024-02-15T03:06:52.295ZMax gen: 188, Max flush: 316
209632024-02-15T03:06:52.295ZError: Difference in extent metadata found!
209642024-02-15T03:06:52.295ZA Difference in extent metadata is expected here
209652024-02-15T03:06:52.295ZOn loop 94
209662024-02-15T03:06:52.295Z
209672024-02-15T03:06:52.295ZVerifying data now
209682024-02-15T03:06:57.547ZNo change after verify
209692024-02-15T03:06:57.547ZLoop: 94 Downstairs dump after verify (and repair):
209702024-02-15T03:06:57.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209712024-02-15T03:06:57.560Z 0 000-019 188 188 188 314 314 314 F F F
209722024-02-15T03:06:57.560Z 1 020-039 187 187 187 312 312 312 F F F
209732024-02-15T03:06:57.560Z 2 040-059 184 184 184 301 301 301 F F F
209742024-02-15T03:06:57.560Z 3 060-079 186 186 186 310 310 310 F F F
209752024-02-15T03:06:57.560Z 4 080-099 188 188 188 314 314 314 F F F
209762024-02-15T03:06:57.560Z 5 100-119 186 186 186 309 309 309 F F F
209772024-02-15T03:06:57.565Z 6 120-139 189 189 189 317 317 317 F F F
209782024-02-15T03:06:57.565Z 7 140-159 188 188 188 316 316 316 F F F
209792024-02-15T03:06:57.565Z 8 160-179 188 188 188 316 316 316 F F F
209802024-02-15T03:06:57.565Z 9 180-199 189 189 189 317 317 317 F F F
209812024-02-15T03:06:57.565Z 10 200-219 187 187 187 312 312 312 F F F
209822024-02-15T03:06:57.565Z 11 220-239 184 184 184 302 302 302 F F F
209832024-02-15T03:06:57.565Z 12 240-259 188 188 188 316 316 316 F F F
209842024-02-15T03:06:57.565Z 13 260-279 182 182 182 300 300 300 F F F
209852024-02-15T03:06:57.565Z 14 280-299 186 186 186 311 311 311 F F F
209862024-02-15T03:06:57.565Z 15 300-319 189 189 189 317 317 317 F F F
209872024-02-15T03:06:57.565Z 16 320-339 181 181 181 299 299 299 F F F
209882024-02-15T03:06:57.565Z 17 340-359 188 188 188 313 313 313 F F F
209892024-02-15T03:06:57.565Z 18 360-379 188 188 188 313 313 313 F F F
209902024-02-15T03:06:57.565Z 19 380-399 186 186 186 307 307 307 F F F
209912024-02-15T03:06:57.565Z 20 400-419 180 180 180 298 298 298 F F F
209922024-02-15T03:06:57.565Z 21 420-439 177 177 177 290 290 290 F F F
209932024-02-15T03:06:57.565Z 22 440-459 177 177 177 290 290 290 F F F
209942024-02-15T03:06:57.565Z 23 460-479 178 178 178 293 293 293 F F F
209952024-02-15T03:06:57.565Z 24 480-499 182 182 182 300 300 300 F F F
209962024-02-15T03:06:57.565Z 25 500-519 188 188 188 316 316 316 F F F
209972024-02-15T03:06:57.565Z 26 520-539 188 188 188 313 313 313 F F F
209982024-02-15T03:06:57.566Z 27 540-559 184 184 184 302 302 302 F F F
209992024-02-15T03:06:57.566Z 28 560-579 182 182 182 300 300 300 F F F
210002024-02-15T03:06:57.566Z 29 580-599 187 187 187 312 312 312 F F F
210012024-02-15T03:06:57.566ZMax gen: 189, Max flush: 317
210022024-02-15T03:06:57.566Z
210032024-02-15T03:06:57.566ZBegin loop 95
210042024-02-15T03:06:57.566ZDownstairs to restart: 2
210052024-02-15T03:06:57.580Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.456475693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904}
210062024-02-15T03:06:57.580Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.456742199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904}
210072024-02-15T03:06:57.580Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.456760597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904}
210082024-02-15T03:06:57.589Z{"msg":"Crucible 3d73a73e-e163-4bf7-ae79-77ce3e2e0670 has session id: e3c85384-5dd7-4bbb-92dd-71ac2f4f416a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.456783164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210092024-02-15T03:06:57.590Z{"msg":"Upstairs opts: Upstairs UUID: 3d73a73e-e163-4bf7-ae79-77ce3e2e0670, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name"Crucible runtime is spawned
210102024-02-15T03:06:57.590Z:"crucible","level":30,"time":"2024-02-15T03:06:56.456837517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210112024-02-15T03:06:57.590Z{"msg":"Crucible stats registered with UUID: 3d73a73e-e163-4bf7-ae79-77ce3e2e0670","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.456877692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210122024-02-15T03:06:57.590Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.456904489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904}
210132024-02-15T03:06:57.590Z{"msg":"3d73a73e-e163-4bf7-ae79-77ce3e2e0670 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457069958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210142024-02-15T03:06:57.590Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457096855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210152024-02-15T03:06:57.591Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457112293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210162024-02-15T03:06:57.591Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457127331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210172024-02-15T03:06:57.591Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457147548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210182024-02-15T03:06:57.591Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457161967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210192024-02-15T03:06:57.591Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457178555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210202024-02-15T03:06:57.591Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457219339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210212024-02-15T03:06:57.591Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457265814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210222024-02-15T03:06:57.591Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.45729601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210232024-02-15T03:06:57.592Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457401127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210242024-02-15T03:06:57.592Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457449631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210252024-02-15T03:06:57.592Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:56.457493755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210262024-02-15T03:06:57.592Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:56.457537959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210272024-02-15T03:06:57.592Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:56.457568356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210282024-02-15T03:06:57.592Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457592603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210292024-02-15T03:06:57.592Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457628378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210302024-02-15T03:06:57.592Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.457712948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210312024-02-15T03:06:57.592Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.458185178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210322024-02-15T03:06:57.592Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.458222984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210332024-02-15T03:06:57.593Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.458239122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210342024-02-15T03:06:57.593Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.45825322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210352024-02-15T03:06:57.593Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.458759506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210362024-02-15T03:06:57.593Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.458807331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210372024-02-15T03:06:57.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.459376219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210382024-02-15T03:06:57.593Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.459419924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210392024-02-15T03:06:57.593Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.459437432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210402024-02-15T03:06:57.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.459459819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210412024-02-15T03:06:57.593Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.459475417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210422024-02-15T03:06:57.593Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:56.459488915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210432024-02-15T03:06:58.583Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.458179162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210442024-02-15T03:06:58.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.45843349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210452024-02-15T03:06:58.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.45931753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210462024-02-15T03:06:58.590Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.459355355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210472024-02-15T03:06:58.590Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.459817817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210482024-02-15T03:06:58.590Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.46042875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210492024-02-15T03:06:58.590Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460476874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210502024-02-15T03:06:58.590Z{"msg":"[0]R flush_numbers[0..12]: [314, 312, 301, 310, 314, 309, 317, 316, 316, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460498251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210512024-02-15T03:06:58.590Z{"msg":"[0]R generation[0..12]: [188, 187, 184, 186, 188, 186, 189, 188, 188, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460531777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210522024-02-15T03:06:58.591Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460565523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210532024-02-15T03:06:58.591Z{"msg":"[1]R flush_numbers[0..12]: [314, 312, 301, 310, 314, 309, 317, 316, 316, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.46059018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210542024-02-15T03:06:58.591Z{"msg":"[1]R generation[0..12]: [188, 187, 184, 186, 188, 186, 189, 188, 188, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460605338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210552024-02-15T03:06:58.591Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460633205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210562024-02-15T03:06:58.591Z{"msg":"[2]R flush_numbers[0..12]: [314, 312, 301, 310, 314, 309, 317, 316, 316, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460649963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210572024-02-15T03:06:58.591Z{"msg":"[2]R generation[0..12]: [188, 187, 184, 186, 188, 186, 189, 188, 188, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460675499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210582024-02-15T03:06:58.591Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460691797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210592024-02-15T03:06:58.591ZWait for a query_work_queue command to finish before sending IO
210602024-02-15T03:06:58.591Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460723753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210612024-02-15T03:06:58.591Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.46075108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210622024-02-15T03:06:58.591Z{"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460771367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210632024-02-15T03:06:58.591Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460793265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"":"downstairs","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210642024-02-15T03:06:58.591Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460807353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210652024-02-15T03:06:58.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460819701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"0","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210662024-02-15T03:06:58.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460840389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"1","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210672024-02-15T03:06:58.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460853837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"client":"2","":"downstairs","session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210682024-02-15T03:06:58.591Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460874355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210692024-02-15T03:06:58.591Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460895242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210702024-02-15T03:06:58.591Z{"msg":"3d73a73e-e163-4bf7-ae79-77ce3e2e0670 is now active with session: e3c85384-5dd7-4bbb-92dd-71ac2f4f416a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460929328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210712024-02-15T03:06:58.591Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460951905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904,"session_id":"e3c85384-5dd7-4bbb-92dd-71ac2f4f416a"}
210722024-02-15T03:06:58.591Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:57.460984971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904}
210732024-02-15T03:06:58.591ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210742024-02-15T03:06:58.591ZLoading write count information from file "/tmp/test_repair_verify.data"
210752024-02-15T03:06:58.591ZRun Repair workload
210762024-02-15T03:06:58.591Z01/30 Write block 485 len 36864 data: 12 15 16 12 15 17 14 16 12
210772024-02-15T03:06:58.592Z02/30 Read block 255 len 32768
210782024-02-15T03:06:58.595Z03/30 Write block 373 len 4096 data: 15
210792024-02-15T03:06:58.595Z04/30 Flush
210802024-02-15T03:06:58.595Z05/30 Write block 84 len 24576 data: 13 13 15 15 12 11
210812024-02-15T03:06:58.595Z06/30 Write block 541 len 8192 data: 14 17
210822024-02-15T03:06:58.595Z07/30 Read block 302 len 40960
210832024-02-15T03:06:58.595Z08/30 Flush
210842024-02-15T03:06:58.599Z09/30 Write block 254 len 36864 data: 17 15 17 19 18 23 20 18 13
210852024-02-15T03:06:58.599Z10/30 Read block 222 len 24576
210862024-02-15T03:06:58.599Z11/30 Read block 228 len 8192
210872024-02-15T03:06:58.599Z12/30 Flush
210882024-02-15T03:06:58.603Z13/30 Write block 357 len 20480 data: 14 15 17 14 12
210892024-02-15T03:06:58.603Z14/30 Flush
210902024-02-15T03:06:58.606Z15/30 Write block 483 len 8192 data: 12 14
210912024-02-15T03:06:58.606Z16/30 Write block 552 len 16384 data: 9 12 14 15
210922024-02-15T03:06:58.606Z17/30 Read block 462 len 32768
210932024-02-15T03:06:58.606Z18/30 Write block 149 len 4096 data: 11
210942024-02-15T03:06:58.606Z19/30 Read block 349 len 24576
210952024-02-15T03:06:58.610Z20/30 Read block 315 len 12288
210962024-02-15T03:06:58.610Z21/30 Read block 296 len 20480
210972024-02-15T03:06:58.610Z22/30 Write block 29 len 32768 data: 13 15 16 15 18 18 19 16
210982024-02-15T03:06:58.610Z23/30 Read block 588 len 24576
210992024-02-15T03:06:58.614Z24/30 Read block 101 len 36864
211002024-02-15T03:06:58.614Z25/30 Write block 451 len 12288 data: 9 8 8
211012024-02-15T03:06:58.614Z26/30 Read block 56 len 28672
211022024-02-15T03:06:58.614Z27/30 Read block 30 len 8192
211032024-02-15T03:06:58.618Z28/30 Read block 37 len 12288
211042024-02-15T03:06:58.618Z29/30 Read block 212 len 40960
211052024-02-15T03:06:58.618Z30/30 Read block 549 len 8192
211062024-02-15T03:06:58.619Z----------------------------------------------------------------
211072024-02-15T03:06:58.619Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
211082024-02-15T03:06:58.619ZStates: Active Active Active
211092024-02-15T03:06:58.619ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211102024-02-15T03:06:58.620Z 1 Acked 1000 Write 9 Done Done Sent false
211112024-02-15T03:06:58.620Z 2 Acked 1001 Read 8 Done Done Sent false
211122024-02-15T03:06:58.620Z 3 Acked 1002 Write 1 Done Done Sent false
211132024-02-15T03:06:58.620Z 4 Acked 1003 Flush 0 Done Done Sent false
211142024-02-15T03:06:58.620Z 5 Acked 1004 Write 6 Done Done Sent false
211152024-02-15T03:06:58.620Z 6 Acked 1005 Write 2 Done Done Sent false
211162024-02-15T03:06:58.621Z 7 Acked 1006 Read 10 Done Done Sent false
211172024-02-15T03:06:58.621Z 8 Acked 1007 Flush 0 Done Done Sent false
211182024-02-15T03:06:58.621Z 9 Acked 1008 Write 9 Done Done Sent false
211192024-02-15T03:06:58.621Z 10 Acked 1009 Read 6 Done Done Sent false
211202024-02-15T03:06:58.621Z 11 Acked 1010 Read 2 Done Done Sent false
211212024-02-15T03:06:58.622Z 12 Acked 1011 Flush 0 Done Done Sent false
211222024-02-15T03:06:58.622Z 13 Acked 1012 Write 5 Done Done Sent false
211232024-02-15T03:06:58.623Z 14 Acked 1013 Flush 0 Done Done Sent false
211242024-02-15T03:06:58.623Z 15 Acked 1014 Write 2 Done Done Sent false
211252024-02-15T03:06:58.623Z 16 Acked 1015 Write 4 Done Done Sent false
211262024-02-15T03:06:58.623Z 17 Acked 1016 Read 8 Done Done Sent false
211272024-02-15T03:06:58.623Z 18 Acked 1017 Write 1 Done Done Sent false
211282024-02-15T03:06:58.623Z 19 Acked 1018 Read 6 Done Done Sent false
211292024-02-15T03:06:58.624Z 20 Acked 1019 Read 3 Done Done Sent false
211302024-02-15T03:06:58.624Z 21 Acked 1020 Read 5 Done Done Sent false
211312024-02-15T03:06:58.624Z 22 Acked 1021 Write 8 Done Done Sent false
211322024-02-15T03:06:58.624Z 23 Acked 1022 Read 6 Done Done Sent false
211332024-02-15T03:06:58.625Z 24 Acked 1023 Read 9 Done Done Sent false
211342024-02-15T03:06:58.625Z 25 Acked 1024 Write 3 Done Done Sent false
211352024-02-15T03:06:58.625Z 26 Acked 1025 Read 7 Done Done Sent false
211362024-02-15T03:06:58.625Z 27 Acked 1026 Read 2 Done Done Sent false
211372024-02-15T03:06:58.625Z 28 Acked 1027 Read 3 Done Done Sent false
211382024-02-15T03:06:58.625Z 29 Acked 1028 Read 10 Done Done Sent false
211392024-02-15T03:06:58.625Z 30 Acked 1029 Read 2 Done Done Sent false
211402024-02-15T03:06:58.625Z STATES DS:0 DS:1 DS:2 TOTAL
211412024-02-15T03:06:58.625Z New 0 0 0 0
211422024-02-15T03:06:58.626Z Sent 0 0 30 30
211432024-02-15T03:06:58.626Z Done 30 30 0 60
211442024-02-15T03:06:58.626Z Skipped 0 0 0 0
211452024-02-15T03:06:58.626Z Error 0 0 0 0
211462024-02-15T03:06:58.626ZLast Flush: 1013 1013 0
211472024-02-15T03:06:58.626ZDownstairs last five completed:
211482024-02-15T03:06:58.626ZUpstairs last five completed: 30 29 28 27 26
211492024-02-15T03:06:58.626Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:57.492984391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1904}
211502024-02-15T03:06:58.626ZWrote out file "/tmp/test_repair_verify.data"
211512024-02-15T03:06:58.633Z
211522024-02-15T03:06:58.633ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211532024-02-15T03:06:58.645ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211542024-02-15T03:06:58.645Z 0 000-019 188 188 188 314 314 314 F F F
211552024-02-15T03:06:58.645Z 1 020-039 187 187 187 312 312 312 T T F <---
211562024-02-15T03:06:58.645Z 2 040-059 184 184 184 301 301 301 F F F
211572024-02-15T03:06:58.645Z 3 060-079 186 186 186 310 310 310 F F F
211582024-02-15T03:06:58.645Z 4 080-099 190 190 188 319 319 314 F F F <---
211592024-02-15T03:06:58.645Z 5 100-119 186 186 186 309 309 309 F F F
211602024-02-15T03:06:58.645Z 6 120-139 189 189 189 317 317 317 F F F
211612024-02-15T03:06:58.646Z 7 140-159 188 188 188 316 316 316 T T F <---
211622024-02-15T03:06:58.649Z 8 160-179 188 188 188 316 316 316 F F F
211632024-02-15T03:06:58.649Z 9 180-199 189 189 189 317 317 317 F F F
211642024-02-15T03:06:58.649Z 10 200-219 187 187 187 312 312 312 F F F
211652024-02-15T03:06:58.650Z 11 220-239 184 184 184 302 302 302 F F F
211662024-02-15T03:06:58.650Z 12 240-259 190 190 188 320 320 316 F F F <---
211672024-02-15T03:06:58.650Z 13 260-279 190 190 182 320 320 300 F F F <---
211682024-02-15T03:06:58.650Z 14 280-299 186 186 186 311 311 311 F F F
211692024-02-15T03:06:58.650Z 15 300-319 189 189 189 317 317 317 F F F
211702024-02-15T03:06:58.650Z 16 320-339 181 181 181 299 299 299 F F F
211712024-02-15T03:06:58.650Z 17 340-359 190 190 188 321 321 313 F F F <---
211722024-02-15T03:06:58.650Z 18 360-379 190 190 188 321 321 313 F F F <---
211732024-02-15T03:06:58.650Z 19 380-399 186 186 186 307 307 307 F F F
211742024-02-15T03:06:58.650Z 20 400-419 180 180 180 298 298 298 F F F
211752024-02-15T03:06:58.650Z 21 420-439 177 177 177 290 290 290 F F F
211762024-02-15T03:06:58.650Z 22 440-459 177 177 177 290 290 290 T T F <---
211772024-02-15T03:06:58.650Z 23 460-479 178 178 178 293 293 293 F F F
211782024-02-15T03:06:58.650Z 24 480-499 190 190 182 318 318 300 T T F <---
211792024-02-15T03:06:58.650Z 25 500-519 188 188 188 316 316 316 F F F
211802024-02-15T03:06:58.650Z 26 520-539 188 188 188 313 313 313 F F F
211812024-02-15T03:06:58.650Z 27 540-559 190 190 184 319 319 302 T T F <---
211822024-02-15T03:06:58.650Z 28 560-579 182 182 182 300 300 300 F F F
211832024-02-15T03:06:58.650Z 29 580-599 187 187 187 312 312 312 F F F
211842024-02-15T03:06:58.650ZMax gen: 190, Max flush: 321
211852024-02-15T03:06:58.651ZError: Difference in extent metadata found!
211862024-02-15T03:06:58.651ZA Difference in extent metadata is expected here
211872024-02-15T03:06:58.651ZOn loop 95
211882024-02-15T03:06:58.651Z
211892024-02-15T03:06:58.651ZVerifying data now
211902024-02-15T03:06:59.178ZNo change after verify
211912024-02-15T03:06:59.179ZLoop: 95 Downstairs dump after verify (and repair):
211922024-02-15T03:06:59.195ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211932024-02-15T03:06:59.195Z 0 000-019 188 188 188 314 314 314 F F F
211942024-02-15T03:06:59.195Z 1 020-039 191 191 191 322 322 322 F F F
211952024-02-15T03:06:59.195Z 2 040-059 184 184 184 301 301 301 F F F
211962024-02-15T03:06:59.196Z 3 060-079 186 186 186 310 310 310 F F F
211972024-02-15T03:06:59.196Z 4 080-099 190 190 190 319 319 319 F F F
211982024-02-15T03:06:59.196Z 5 100-119 186 186 186 309 309 309 F F F
211992024-02-15T03:06:59.196Z 6 120-139 189 189 189 317 317 317 F F F
212002024-02-15T03:06:59.196Z 7 140-159 191 191 191 322 322 322 F F F
212012024-02-15T03:06:59.197Z 8 160-179 188 188 188 316 316 316 F F F
212022024-02-15T03:06:59.197Z 9 180-199 189 189 189 317 317 317 F F F
212032024-02-15T03:06:59.211Z 10 200-219 187 187 187 312 312 312 F F F
212042024-02-15T03:06:59.211Z 11 220-239 184 184 184 302 302 302 F F F
212052024-02-15T03:06:59.211Z 12 240-259 190 190 190 320 320 320 F F F
212062024-02-15T03:06:59.211Z 13 260-279 190 190 190 320 320 320 F F F
212072024-02-15T03:06:59.211Z 14 280-299 186 186 186 311 311 311 F F F
212082024-02-15T03:06:59.211Z 15 300-319 189 189 189 317 317 317 F F F
212092024-02-15T03:06:59.212Z 16 320-339 181 181 181 299 299 299 F F F
212102024-02-15T03:06:59.212Z 17 340-359 190 190 190 321 321 321 F F F
212112024-02-15T03:06:59.212Z 18 360-379 190 190 190 321 321 321 F F F
212122024-02-15T03:06:59.212Z 19 380-399 186 186 186 307 307 307 F F F
212132024-02-15T03:06:59.212Z 20 400-419 180 180 180 298 298 298 F F F
212142024-02-15T03:06:59.212Z 21 420-439 177 177 177 290 290 290 F F F
212152024-02-15T03:06:59.212Z 22 440-459 191 191 191 322 322 322 F F F
212162024-02-15T03:06:59.212Z 23 460-479 178 178 178 293 293 293 F F F
212172024-02-15T03:06:59.212Z 24 480-499 191 191 191 322 322 322 F F F
212182024-02-15T03:06:59.212Z 25 500-519 188 188 188 316 316 316 F F F
212192024-02-15T03:06:59.212Z 26 520-539 188 188 188 313 313 313 F F F
212202024-02-15T03:06:59.212Z 27 540-559 191 191 191 322 322 322 F F F
212212024-02-15T03:06:59.213Z 28 560-579 182 182 182 300 300 300 F F F
212222024-02-15T03:06:59.213Z 29 580-599 187 187 187 312 312 312 F F F
212232024-02-15T03:06:59.213ZMax gen: 191, Max flush: 322
212242024-02-15T03:06:59.213Z
212252024-02-15T03:06:59.213ZBegin loop 96
212262024-02-15T03:06:59.213ZDownstairs to restart: 0
212272024-02-15T03:06:59.219Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.094651074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912}
212282024-02-15T03:06:59.219Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"Crucible runtime is spawned
212292024-02-15T03:06:59.219Z2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.094979733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912}
212302024-02-15T03:06:59.225Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095031017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912}
212312024-02-15T03:06:59.226Z{"msg":"Crucible ac58c736-3256-4d72-b505-3ad8deb9e96d has session id: c7c5fae5-dc88-4dee-b8a3-90ababe57723","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095062093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212322024-02-15T03:06:59.226Z{"msg":"Upstairs opts: Upstairs UUID: ac58c736-3256-4d72-b505-3ad8deb9e96d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095130574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212332024-02-15T03:06:59.226Z{"msg":"Crucible stats registered with UUID: ac58c736-3256-4d72-b505-3ad8deb9e96d","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095213894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212342024-02-15T03:06:59.226Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095274196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912}
212352024-02-15T03:06:59.227Z{"msg":"ac58c736-3256-4d72-b505-3ad8deb9e96d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095316411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212362024-02-15T03:06:59.227Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095357676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212372024-02-15T03:06:59.227Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095393671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212382024-02-15T03:06:59.227Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095431476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212392024-02-15T03:06:59.227Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095469642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212402024-02-15T03:06:59.227Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095513706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212412024-02-15T03:06:59.227Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095565219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212422024-02-15T03:06:59.228Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095600295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212432024-02-15T03:06:59.228Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095651879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212442024-02-15T03:06:59.228Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095694943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212452024-02-15T03:06:59.228Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.09572159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212462024-02-15T03:06:59.228Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.095765545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212472024-02-15T03:06:59.228Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:58.095914046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212482024-02-15T03:06:59.228Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:58.095969809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212492024-02-15T03:06:59.228Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:58.096033851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212502024-02-15T03:06:59.229Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.096074356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212512024-02-15T03:06:59.229Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.096113891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212522024-02-15T03:06:59.229Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.096142917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212532024-02-15T03:06:59.229Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.096553146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212542024-02-15T03:06:59.230Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.096622687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212552024-02-15T03:06:59.230Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.096668621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212562024-02-15T03:06:59.230Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.096694588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212572024-02-15T03:06:59.230Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.097185647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212582024-02-15T03:06:59.230Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.09732005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212592024-02-15T03:06:59.230Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.097696372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212602024-02-15T03:06:59.230Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.097765754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212612024-02-15T03:06:59.230Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.097801449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212622024-02-15T03:06:59.230Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.097922574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212632024-02-15T03:06:59.231Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.09795604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212642024-02-15T03:06:59.231Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:58.097972028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212652024-02-15T03:07:00.223Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.097669309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212662024-02-15T03:07:00.223Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.097764377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212672024-02-15T03:07:00.229Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.098713528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212682024-02-15T03:07:00.229Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.098759042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212692024-02-15T03:07:00.229Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099277417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212702024-02-15T03:07:00.229Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099823809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212712024-02-15T03:07:00.229Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099851636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212722024-02-15T03:07:00.229Z{"msg":"[0]R flush_numbers[0..12]: [314, 322, 301, 310, 319, 309, 317, 322, 316, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099867674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212732024-02-15T03:07:00.230Z{"msg":"[0]R generation[0..12]: [188, 191, 184, 186, 190, 186, 189, 191, 188, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099882482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212742024-02-15T03:07:00.230Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099905079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212752024-02-15T03:07:00.230Z{"msg":"[1]R flush_numbers[0..12]: [314, 322, 301, 310, 319, 309, 317, 322, 316, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099920607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212762024-02-15T03:07:00.230Z{"msg":"[1]R generation[0..12]: [188, 191, 184, 186, 190, 186, 189, 191, 188, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099943484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212772024-02-15T03:07:00.230Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099958582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212782024-02-15T03:07:00.230Z{"msg":"[2]R flush_numbers[0..12]: [314, 322, 301, 310, 319, 309, 317, 322, 316, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.09997287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212792024-02-15T03:07:00.230Z{"msg":"[2]R generation[0..12]: [188, 191, 184, 186, 190, 186, 189, 191, 188, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.099988308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212802024-02-15T03:07:00.230Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100010756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212812024-02-15T03:07:00.231Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100024754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212822024-02-15T03:07:00.231Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100038662Z"Wait for a query_work_queue command to finish before sending IO
212832024-02-15T03:07:00.231Z,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212842024-02-15T03:07:00.231Z{"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100064489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212852024-02-15T03:07:00.231Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100077867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"":"downstairs","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212862024-02-15T03:07:00.231Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100092705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212872024-02-15T03:07:00.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100105054Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"0","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212882024-02-15T03:07:00.231Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100125561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"1","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212892024-02-15T03:07:00.232Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100139329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"client":"2","":"downstairs","session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212902024-02-15T03:07:00.232Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100159557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212912024-02-15T03:07:00.232Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100172145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212922024-02-15T03:07:00.232Z{"msg":"ac58c736-3256-4d72-b505-3ad8deb9e96d is now active with session: c7c5fae5-dc88-4dee-b8a3-90ababe57723","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100186303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212932024-02-15T03:07:00.232Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.100199762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912,"session_id":"c7c5fae5-dc88-4dee-b8a3-90ababe57723"}
212942024-02-15T03:07:00.232Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:06:59.10021228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912}
212952024-02-15T03:07:00.232ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212962024-02-15T03:07:00.232ZLoading write count information from file "/tmp/test_repair_verify.data"
212972024-02-15T03:07:00.232ZRun Repair workload
212982024-02-15T03:07:00.232Z01/30 Write block 298 len 28672 data: 14 16 16 15 14 16 17
212992024-02-15T03:07:00.256Z02/30 Write block 460 len 4096 data: 15
213002024-02-15T03:07:00.256Z03/30 Write block 280 len 8192 data: 15 14
213012024-02-15T03:07:00.256Z04/30 Read block 285 len 16384
213022024-02-15T03:07:00.256Z05/30 Write block 384 len 24576 data: 16 16 15 16 17 19
213032024-02-15T03:07:00.256Z06/30 Write block 170 len 12288 data: 10 11 9
213042024-02-15T03:07:00.257Z07/30 Write block 389 len 16384 data: 20 17 16 14
213052024-02-15T03:07:00.257Z08/30 Write block 119 len 36864 data: 8 9 9 8 9 10 12 14 14
213062024-02-15T03:07:00.257Z09/30 Write block 100 len 32768 data: 14 16 15 13 14 11 9 9
213072024-02-15T03:07:00.257Z10/30 Read block 86 len 40960
213082024-02-15T03:07:00.257Z11/30 Read block 547 len 16384
213092024-02-15T03:07:00.257Z12/30 Write block 348 len 8192 data: 15 16
213102024-02-15T03:07:00.258Z13/30 Write block 171 len 20480 data: 12 10 7 7 8
213112024-02-15T03:07:00.258Z14/30 Read block 15 len 28672
213122024-02-15T03:07:00.271Z15/30 Write block 74 len 32768 data: 11 14 16 14 12 10 9 10
213132024-02-15T03:07:00.271Z16/30 Write block 138 len 24576 data: 16 14 16 18 16 14
213142024-02-15T03:07:00.271Z17/30 Write block 517 len 12288 data: 14 14 13
213152024-02-15T03:07:00.272Z18/30 Write block 439 len 24576 data: 13 15 15 16 14 12
213162024-02-15T03:07:00.272Z19/30 Write block 66 len 40960 data: 15 16 16 15 14 15 15 9 12 15
213172024-02-15T03:07:00.272Z20/30 Write block 3 len 40960 data: 10 13 13 14 13 15 13 18 13 11
213182024-02-15T03:07:00.273Z21/30 Read block 69 len 12288
213192024-02-15T03:07:00.273Z22/30 Read block 282 len 40960
213202024-02-15T03:07:00.273Z23/30 Read block 482 len 4096
213212024-02-15T03:07:00.273Z24/30 Write block 78 len 36864 data: 13 11 10 11 10 10 14 14 16
213222024-02-15T03:07:00.273Z25/30 Write block 113 len 32768 data: 8 9 11 11 12 10 9 10
213232024-02-15T03:07:00.274Z26/30 Read block 1 len 40960
213242024-02-15T03:07:00.274Z27/30 Read block 228 len 4096
213252024-02-15T03:07:00.274Z28/30 Read block 49 len 20480
213262024-02-15T03:07:00.274Z29/30 Write block 567 len 8192 data: 14 13
213272024-02-15T03:07:00.274Z30/30 Write block 336 len 16384 data: 17 17 17 13
213282024-02-15T03:07:00.275Z----------------------------------------------------------------
213292024-02-15T03:07:00.275Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
213302024-02-15T03:07:00.275ZStates: Active Active Active
213312024-02-15T03:07:00.275ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213322024-02-15T03:07:00.275Z 1 Acked 1000 Write 7 Done Done Done false
213332024-02-15T03:07:00.275Z 2 Acked 1001 Write 1 Done Done Done false
213342024-02-15T03:07:00.275Z 3 Acked 1002 Write 2 Done Done Done false
213352024-02-15T03:07:00.275Z 4 Acked 1003 Read 4 Done Done Done false
213362024-02-15T03:07:00.275Z 5 Acked 1004 Write 6 Done Done Done false
213372024-02-15T03:07:00.276Z 6 Acked 1005 Write 3 Done Done Done false
213382024-02-15T03:07:00.276Z 7 Acked 1006 Write 4 Done Done Done false
213392024-02-15T03:07:00.276Z 8 Acked 1007 Write 9 Done Done Done false
213402024-02-15T03:07:00.276Z 9 Acked 1008 Write 8 Done Done Done false
213412024-02-15T03:07:00.276Z 10 Acked 1009 Read 10 Done Done Done false
213422024-02-15T03:07:00.276Z 11 Acked 1010 Read 4 Done Done Done false
213432024-02-15T03:07:00.276Z 12 Acked 1011 Write 2 Sent Done Done false
213442024-02-15T03:07:00.276Z 13 Acked 1012 Write 5 Sent Done Done false
213452024-02-15T03:07:00.276Z 14 Acked 1013 Read 7 Sent Done Done false
213462024-02-15T03:07:00.276Z 15 Acked 1014 Write 8 Sent Done Done false
213472024-02-15T03:07:00.276Z 16 Acked 1015 Write 6 Sent Done Done false
213482024-02-15T03:07:00.277Z 17 Acked 1016 Write 3 Sent Done Done false
213492024-02-15T03:07:00.277Z 18 Acked 1017 Write 6 Sent Done Done false
213502024-02-15T03:07:00.277Z 19 Acked 1018 Write 10 Sent Done Done false
213512024-02-15T03:07:00.277Z 20 Acked 1019 Write 10 Sent Done Done false
213522024-02-15T03:07:00.277Z 21 Acked 1020 Read 3 Sent Done Done false
213532024-02-15T03:07:00.277Z 22 Acked 1021 Read 10 Sent Done Done false
213542024-02-15T03:07:00.277Z 23 Acked 1022 Read 1 Sent Done Done false
213552024-02-15T03:07:00.277Z 24 Acked 1023 Write 9 Sent Done Done false
213562024-02-15T03:07:00.277Z 25 Acked 1024 Write 8 Sent Done Done false
213572024-02-15T03:07:00.277Z 26 Acked 1025 Read 10 Sent Done Done false
213582024-02-15T03:07:00.278Z 27 Acked 1026 Read 1 Sent Done Done false
213592024-02-15T03:07:00.278Z 28 Acked 1027 Read 5 Sent Done Done false
213602024-02-15T03:07:00.278Z 29 Acked 1028 Write 2 Sent Done Done false
213612024-02-15T03:07:00.278Z 30 Acked 1029 Write 4 Sent Sent Sent false
213622024-02-15T03:07:00.278Z STATES DS:0 DS:1 DS:2 TOTAL
213632024-02-15T03:07:00.278Z New 0 0 0 0
213642024-02-15T03:07:00.279Z Sent 19 1 1 21
213652024-02-15T03:07:00.279Z Done 11 29 29 69
213662024-02-15T03:07:00.279Z Skipped 0 0 0 0
213672024-02-15T03:07:00.279Z Error 0 0 0 0
213682024-02-15T03:07:00.279ZLast Flush: 0 0 0
213692024-02-15T03:07:00.279ZDownstairs last five completed:
213702024-02-15T03:07:00.279ZUpstairs last five completed: 30 29 28 27 26
213712024-02-15T03:07:00.279Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:06:59.127713905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1912}
213722024-02-15T03:07:00.280ZWrote out file "/tmp/test_repair_verify.data"
213732024-02-15T03:07:00.280Z
213742024-02-15T03:07:00.280ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213752024-02-15T03:07:00.286ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213762024-02-15T03:07:00.286Z 0 000-019 188 188 188 314 314 314 F T T <---
213772024-02-15T03:07:00.286Z 1 020-039 191 191 191 322 322 322 F F F
213782024-02-15T03:07:00.286Z 2 040-059 184 184 184 301 301 301 F F F
213792024-02-15T03:07:00.287Z 3 060-079 186 186 186 310 310 310 F T T <---
213802024-02-15T03:07:00.287Z 4 080-099 190 190 190 319 319 319 F T T <---
213812024-02-15T03:07:00.287Z 5 100-119 186 186 186 309 309 309 T T T
213822024-02-15T03:07:00.287Z 6 120-139 189 189 189 317 317 317 T T T
213832024-02-15T03:07:00.287Z 7 140-159 191 191 191 322 322 322 F T T <---
213842024-02-15T03:07:00.287Z 8 160-179 188 188 188 316 316 316 T T T
213852024-02-15T03:07:00.287Z 9 180-199 189 189 189 317 317 317 F F F
213862024-02-15T03:07:00.287Z 10 200-219 187 187 187 312 312 312 F F F
213872024-02-15T03:07:00.287Z 11 220-239 184 184 184 302 302 302 F F F
213882024-02-15T03:07:00.287Z 12 240-259 190 190 190 320 320 320 F F F
213892024-02-15T03:07:00.287Z 13 260-279 190 190 190 320 320 320 F F F
213902024-02-15T03:07:00.287Z 14 280-299 186 186 186 311 311 311 T T T
213912024-02-15T03:07:00.288Z 15 300-319 189 189 189 317 317 317 T T T
213922024-02-15T03:07:00.288Z 16 320-339 181 181 181 299 299 299 F T T <---
213932024-02-15T03:07:00.288Z 17 340-359 190 190 190 321 321 321 F T T <---
213942024-02-15T03:07:00.288Z 18 360-379 190 190 190 321 321 321 F F F
213952024-02-15T03:07:00.288Z 19 380-399 186 186 186 307 307 307 T T T
213962024-02-15T03:07:00.288Z 20 400-419 180 180 180 298 298 298 F F F
213972024-02-15T03:07:00.288Z 21 420-439 177 177 177 290 290 290 F T T <---
213982024-02-15T03:07:00.288Z 22 440-459 191 191 191 322 322 322 F T T <---
213992024-02-15T03:07:00.288Z 23 460-479 178 178 178 293 293 293 T T T
214002024-02-15T03:07:00.288Z 24 480-499 191 191 191 322 322 322 F F F
214012024-02-15T03:07:00.289Z 25 500-519 188 188 188 316 316 316 F T T <---
214022024-02-15T03:07:00.289Z 26 520-539 188 188 188 313 313 313 F F F
214032024-02-15T03:07:00.289Z 27 540-559 191 191 191 322 322 322 F F F
214042024-02-15T03:07:00.289Z 28 560-579 182 182 182 300 300 300 F T T <---
214052024-02-15T03:07:00.289Z 29 580-599 187 187 187 312 312 312 F F F
214062024-02-15T03:07:00.289ZMax gen: 191, Max flush: 322
214072024-02-15T03:07:00.296ZError: Difference in extent metadata found!
214082024-02-15T03:07:00.296ZA Difference in extent metadata is expected here
214092024-02-15T03:07:00.296ZOn loop 96
214102024-02-15T03:07:00.296Z
214112024-02-15T03:07:00.296ZVerifying data now
214122024-02-15T03:07:06.257ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
214132024-02-15T03:07:06.262Z116d115
214142024-02-15T03:07:06.262Z< 7,
214152024-02-15T03:07:06.262Z117a117,120
214162024-02-15T03:07:06.262Z> 9,
214172024-02-15T03:07:06.263Z> 11,
214182024-02-15T03:07:06.263Z> 11,
214192024-02-15T03:07:06.263Z> 12,
214202024-02-15T03:07:06.263Z118a122
214212024-02-15T03:07:06.263Z> 9,
214222024-02-15T03:07:06.263Z120d123
214232024-02-15T03:07:06.263Z< 11,
214242024-02-15T03:07:06.263Z124,126d126
214252024-02-15T03:07:06.263Z< 9,
214262024-02-15T03:07:06.263Z< 8,
214272024-02-15T03:07:06.263Z< 9,
214282024-02-15T03:07:06.263Z141,142d140
214292024-02-15T03:07:06.263Z< 15,
214302024-02-15T03:07:06.264Z< 13,
214312024-02-15T03:07:06.264Z143a142,143
214322024-02-15T03:07:06.264Z> 14,
214332024-02-15T03:07:06.264Z> 16,
214342024-02-15T03:07:06.264Z174,177c174,175
214352024-02-15T03:07:06.264Z< 11,
214362024-02-15T03:07:06.264Z< 9,
214372024-02-15T03:07:06.264Z< 6,
214382024-02-15T03:07:06.264Z< 6,
214392024-02-15T03:07:06.264Z---
214402024-02-15T03:07:06.264Z> 12,
214412024-02-15T03:07:06.264Z> 10,
214422024-02-15T03:07:06.264Z179a178,179
214432024-02-15T03:07:06.264Z> 8,
214442024-02-15T03:07:06.264Z> 7,
214452024-02-15T03:07:06.264Zdiff found after verify
214462024-02-15T03:07:06.264ZLoop: 96 Downstairs dump after verify (and repair):
214472024-02-15T03:07:06.283ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214482024-02-15T03:07:06.283Z 0 000-019 192 192 192 323 323 323 F F F
214492024-02-15T03:07:06.283Z 1 020-039 191 191 191 322 322 322 F F F
214502024-02-15T03:07:06.283Z 2 040-059 184 184 184 301 301 301 F F F
214512024-02-15T03:07:06.283Z 3 060-079 192 192 192 323 323 323 F F F
214522024-02-15T03:07:06.284Z 4 080-099 192 192 192 323 323 323 F F F
214532024-02-15T03:07:06.284Z 5 100-119 192 192 192 323 323 323 F F F
214542024-02-15T03:07:06.284Z 6 120-139 192 192 192 323 323 323 F F F
214552024-02-15T03:07:06.284Z 7 140-159 192 192 192 323 323 323 F F F
214562024-02-15T03:07:06.284Z 8 160-179 192 192 192 323 323 323 F F F
214572024-02-15T03:07:06.284Z 9 180-199 189 189 189 317 317 317 F F F
214582024-02-15T03:07:06.284Z 10 200-219 187 187 187 312 312 312 F F F
214592024-02-15T03:07:06.284Z 11 220-239 184 184 184 302 302 302 F F F
214602024-02-15T03:07:06.284Z 12 240-259 190 190 190 320 320 320 F F F
214612024-02-15T03:07:06.284Z 13 260-279 190 190 190 320 320 320 F F F
214622024-02-15T03:07:06.284Z 14 280-299 192 192 192 323 323 323 F F F
214632024-02-15T03:07:06.284Z 15 300-319 192 192 192 323 323 323 F F F
214642024-02-15T03:07:06.284Z 16 320-339 192 192 192 323 323 323 F F F
214652024-02-15T03:07:06.284Z 17 340-359 192 192 192 323 323 323 F F F
214662024-02-15T03:07:06.284Z 18 360-379 190 190 190 321 321 321 F F F
214672024-02-15T03:07:06.284Z 19 380-399 192 192 192 323 323 323 F F F
214682024-02-15T03:07:06.284Z 20 400-419 180 180 180 298 298 298 F F F
214692024-02-15T03:07:06.297Z 21 420-439 192 192 192 323 323 323 F F F
214702024-02-15T03:07:06.297Z 22 440-459 192 192 192 323 323 323 F F F
214712024-02-15T03:07:06.297Z 23 460-479 192 192 192 323 323 323 F F F
214722024-02-15T03:07:06.297Z 24 480-499 191 191 191 322 322 322 F F F
214732024-02-15T03:07:06.297Z 25 500-519 192 192 192 323 323 323 F F F
214742024-02-15T03:07:06.297Z 26 520-539 188 188 188 313 313 313 F F F
214752024-02-15T03:07:06.297Z 27 540-559 191 191 191 322 322 322 F F F
214762024-02-15T03:07:06.297Z 28 560-579 192 192 192 323 323 323 F F F
214772024-02-15T03:07:06.297Z 29 580-599 187 187 187 312 312 312 F F F
214782024-02-15T03:07:06.297ZMax gen: 192, Max flush: 323
214792024-02-15T03:07:06.297Z
214802024-02-15T03:07:06.297ZBegin loop 97
214812024-02-15T03:07:06.298ZDownstairs to restart: 0
214822024-02-15T03:07:06.304Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.178553353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923}
214832024-02-15T03:07:06.304Z{Crucible runtime is spawned
214842024-02-15T03:07:06.304Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.178938375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923}
214852024-02-15T03:07:06.316Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.178971251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923}
214862024-02-15T03:07:06.316Z{"msg":"Crucible 0a738db9-2cff-468f-b63e-7fe5e3b9043a has session id: 33062a72-f6fa-42f7-abd7-1df12ad1d797","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.178995198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214872024-02-15T03:07:06.316Z{"msg":"Upstairs opts: Upstairs UUID: 0a738db9-2cff-468f-b63e-7fe5e3b9043a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179033013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214882024-02-15T03:07:06.316Z{"msg":"Crucible stats registered with UUID: 0a738db9-2cff-468f-b63e-7fe5e3b9043a","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179077708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214892024-02-15T03:07:06.317Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179103964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923}
214902024-02-15T03:07:06.317Z{"msg":"0a738db9-2cff-468f-b63e-7fe5e3b9043a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.17921396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214912024-02-15T03:07:06.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179248396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214922024-02-15T03:07:06.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179265284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214932024-02-15T03:07:06.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.17929289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214942024-02-15T03:07:06.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179308788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214952024-02-15T03:07:06.317Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179329796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214962024-02-15T03:07:06.317Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179364732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214972024-02-15T03:07:06.317Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179402897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214982024-02-15T03:07:06.317Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179435683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
214992024-02-15T03:07:06.317Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179480817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215002024-02-15T03:07:06.317Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:05.179525871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215012024-02-15T03:07:06.317Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:05.179556148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215022024-02-15T03:07:06.317Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179581085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215032024-02-15T03:07:06.317Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179625449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215042024-02-15T03:07:06.317Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:05.179677952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215052024-02-15T03:07:06.317Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.17969599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215062024-02-15T03:07:06.317Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179718607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215072024-02-15T03:07:06.317Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.179924491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215082024-02-15T03:07:06.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.180465614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215092024-02-15T03:07:06.317Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.180503159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215102024-02-15T03:07:06.317Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.180520507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215112024-02-15T03:07:06.317Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.180534655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215122024-02-15T03:07:06.317Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.181098924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215132024-02-15T03:07:06.318Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.181137399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215142024-02-15T03:07:06.318Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.181722816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215152024-02-15T03:07:06.318Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.181775239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215162024-02-15T03:07:06.318Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.181798876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215172024-02-15T03:07:06.318Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.181822244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215182024-02-15T03:07:06.318Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.181837092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215192024-02-15T03:07:06.318Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:05.18185152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215202024-02-15T03:07:07.309Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.180352181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215212024-02-15T03:07:07.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.180449029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215222024-02-15T03:07:07.341Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.181291593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215232024-02-15T03:07:07.342Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.181329468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215242024-02-15T03:07:07.342Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.181835755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215252024-02-15T03:07:07.342Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182403284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215262024-02-15T03:07:07.342Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182441279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215272024-02-15T03:07:07.342Z{"msg":"[0]R flush_numbers[0..12]: [323, 322, 301, 323, 323, 323, 323, 323, 323, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182470745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215282024-02-15T03:07:07.342Z{"msg":"[0]R generation[0..12]: [192, 191, 184, 192, 192, 192, 192, 192, 192, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182507251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215292024-02-15T03:07:07.342Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182524709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215302024-02-15T03:07:07.342Z{"msg":"[1]R flush_numbers[0..12]: [323, 322, 301, 323, 323, 323, 323, 323, 323, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182546726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215312024-02-15T03:07:07.342Z{"msg":"[1]R generation[0..12]: [192, 191, 184, 192, 192, 192, 192, 192, 192, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182562034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215322024-02-15T03:07:07.342Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182583161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215332024-02-15T03:07:07.342Z{"msg":"[2]R flush_numbers[0..12]: [323, 322, 301, 323, 323, 323, 323, 323, 323, 317, 312, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182599389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215342024-02-15T03:07:07.343Z{"msg":"[2]R generation[0..12]: [192, 191, 184, 192, 192, 192, 192, 192, 192, 189, 187, 184]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
215352024-02-15T03:07:07.343Z,"time":"2024-02-15T03:07:06.182632545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215362024-02-15T03:07:07.343Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.18267069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215372024-02-15T03:07:07.343Z{"msg":"Max found gen is 193","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182699717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215382024-02-15T03:07:07.343Z{"msg":"Generation requested: 194 >= found:193","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182737102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215392024-02-15T03:07:07.343Z{"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.18275617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215402024-02-15T03:07:07.343Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182784336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"":"downstairs","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215412024-02-15T03:07:07.343Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182811163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215422024-02-15T03:07:07.343Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182844208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"0","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215432024-02-15T03:07:07.343Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182865896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"1","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215442024-02-15T03:07:07.343Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182902861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"client":"2","":"downstairs","session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215452024-02-15T03:07:07.343Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182927468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215462024-02-15T03:07:07.344Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182961454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215472024-02-15T03:07:07.344Z{"msg":"0a738db9-2cff-468f-b63e-7fe5e3b9043a is now active with session: 33062a72-f6fa-42f7-abd7-1df12ad1d797","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.182984431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215482024-02-15T03:07:07.344Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.183000819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923,"session_id":"33062a72-f6fa-42f7-abd7-1df12ad1d797"}
215492024-02-15T03:07:07.344Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:06.183031195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923}
215502024-02-15T03:07:07.344ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
215512024-02-15T03:07:07.344ZLoading write count information from file "/tmp/test_repair_verify.data"
215522024-02-15T03:07:07.344ZRun Repair workload
215532024-02-15T03:07:07.344Z01/30 Write block 565 len 8192 data: 17 14
215542024-02-15T03:07:07.344Z02/30 Read block 578 len 16384
215552024-02-15T03:07:07.344Z03/30 Write block 395 len 8192 data: 13 15
215562024-02-15T03:07:07.344Z04/30 Write block 591 len 28672 data: 19 15 16 15 14 9 7
215572024-02-15T03:07:07.344Z05/30 Flush
215582024-02-15T03:07:07.344Z06/30 Write block 330 len 32768 data: 16 18 17 17 20 19 18 18
215592024-02-15T03:07:07.345Z07/30 Read block 344 len 4096
215602024-02-15T03:07:07.345Z08/30 Write block 92 len 32768 data: 11 9 13 13 16 19 19 17
215612024-02-15T03:07:07.345Z09/30 Read block 456 len 28672
215622024-02-15T03:07:07.345Z10/30 Read block 356 len 16384
215632024-02-15T03:07:07.345Z11/30 Write block 218 len 32768 data: 13 15 16 17 17 16 15 15
215642024-02-15T03:07:07.345Z12/30 Read block 110 len 16384
215652024-02-15T03:07:07.345Z13/30 Read block 232 len 8192
215662024-02-15T03:07:07.345Z14/30 Read block 149 len 8192
215672024-02-15T03:07:07.345Z15/30 Read block 171 len 40960
215682024-02-15T03:07:07.345Z16/30 Write block 344 len 20480 data: 14 14 16 17 16
215692024-02-15T03:07:07.360Z17/30 Write block 146 len 28672 data: 13 13 10 12 12 12 11
215702024-02-15T03:07:07.360Z18/30 Read block 175 len 28672
215712024-02-15T03:07:07.360Z19/30 Write block 491 len 12288 data: 15 17 13
215722024-02-15T03:07:07.361Z20/30 Write block 254 len 40960 data: 18 16 18 20 19 24 21 19 14 12
215732024-02-15T03:07:07.361Z21/30 Write block 239 len 40960 data: 16 16 18 21 16 14 14 12 14 12
215742024-02-15T03:07:07.361Z22/30 Write block 353 len 4096 data: 18
215752024-02-15T03:07:07.361Z23/30 Read block 347 len 40960
215762024-02-15T03:07:07.361Z24/30 Write block 236 len 32768 data: 18 17 16 17 17 19 22 17
215772024-02-15T03:07:07.361Z25/30 Flush
215782024-02-15T03:07:07.361Z26/30 Flush
215792024-02-15T03:07:07.361Z27/30 Write block 325 len 32768 data: 15 16 14 13 15 17 19 18
215802024-02-15T03:07:07.361Z28/30 Read block 451 len 12288
215812024-02-15T03:07:07.361Z29/30 Read block 120 len 4096
215822024-02-15T03:07:07.361Z30/30 Read block 539 len 20480
215832024-02-15T03:07:07.361Z----------------------------------------------------------------
215842024-02-15T03:07:07.361Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
215852024-02-15T03:07:07.362ZStates: Active Active Active
215862024-02-15T03:07:07.362ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
215872024-02-15T03:07:07.362Z 1 Acked 1000 Write 2 Done Done Done false
215882024-02-15T03:07:07.362Z 2 Acked 1001 Read 4 Sent Done Done false
215892024-02-15T03:07:07.362Z 3 Acked 1002 Write 2 Sent Done Done false
215902024-02-15T03:07:07.362Z 4 Acked 1003 Write 7 Sent Done Done false
215912024-02-15T03:07:07.362Z 5 Acked 1004 Flush 0 Sent Done Done false
215922024-02-15T03:07:07.362Z 6 Acked 1005 Write 8 Sent Done Done false
215932024-02-15T03:07:07.362Z 7 Acked 1006 Read 1 Sent Done Done false
215942024-02-15T03:07:07.362Z 8 Acked 1007 Write 8 Sent Done Done false
215952024-02-15T03:07:07.362Z 9 Acked 1008 Read 7 Sent Done Done false
215962024-02-15T03:07:07.362Z 10 Acked 1009 Read 4 Sent Done Done false
215972024-02-15T03:07:07.362Z 11 Acked 1010 Write 8 Sent Done Done false
215982024-02-15T03:07:07.362Z 12 Acked 1011 Read 4 Sent Done Done false
215992024-02-15T03:07:07.363Z 13 Acked 1012 Read 2 Sent Done Done false
216002024-02-15T03:07:07.363Z 14 Acked 1013 Read 2 Sent Done Done false
216012024-02-15T03:07:07.363Z 15 Acked 1014 Read 10 Sent Done Done false
216022024-02-15T03:07:07.363Z 16 Acked 1015 Write 5 Sent Done Done false
216032024-02-15T03:07:07.363Z 17 Acked 1016 Write 7 Sent Done Done false
216042024-02-15T03:07:07.363Z 18 Acked 1017 Read 7 Sent Done Done false
216052024-02-15T03:07:07.363Z 19 Acked 1018 Write 3 Sent Done Done false
216062024-02-15T03:07:07.363Z 20 Acked 1019 Write 10 Sent Done Done false
216072024-02-15T03:07:07.363Z 21 Acked 1020 Write 10 Sent Done Done false
216082024-02-15T03:07:07.363Z 22 Acked 1021 Write 1 Sent Done Done false
216092024-02-15T03:07:07.363Z 23 Acked 1022 Read 10 Sent Done Done false
216102024-02-15T03:07:07.363Z 24 Acked 1023 Write 8 Sent Done Done false
216112024-02-15T03:07:07.363Z 25 Acked 1024 Flush 0 Sent Done Done false
216122024-02-15T03:07:07.363Z 26 Acked 1025 Flush 0 Sent Done Done false
216132024-02-15T03:07:07.363Z 27 Acked 1026 Write 8 Sent Done Done false
216142024-02-15T03:07:07.363Z 28 Acked 1027 Read 3 Sent Done Done false
216152024-02-15T03:07:07.363Z 29 Acked 1028 Read 1 Sent Done Done false
216162024-02-15T03:07:07.363Z 30 Acked 1029 Read 5 Sent Done Done false
216172024-02-15T03:07:07.363Z STATES DS:0 DS:1 DS:2 TOTAL
216182024-02-15T03:07:07.363Z New 0 0 0 0
216192024-02-15T03:07:07.363Z Sent 29 0 0 29
216202024-02-15T03:07:07.363Z Done 1 30 30 61
216212024-02-15T03:07:07.363Z Skipped 0 0 0 0
216222024-02-15T03:07:07.363Z Error 0 0 0 0
216232024-02-15T03:07:07.364ZLast Flush: 0 1025 1025
216242024-02-15T03:07:07.364ZDownstairs last five completed:
216252024-02-15T03:07:07.364ZUpstairs last five completed: 30 29 28 27 26
216262024-02-15T03:07:07.364Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:06.216853387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1923}
216272024-02-15T03:07:07.364ZWrote out file "/tmp/test_repair_verify.data"
216282024-02-15T03:07:07.364Z
216292024-02-15T03:07:07.364ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
216302024-02-15T03:07:07.369ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
216312024-02-15T03:07:07.369Z 0 000-019 192 192 192 323 323 323 F F F
216322024-02-15T03:07:07.369Z 1 020-039 191 191 191 322 322 322 F F F
216332024-02-15T03:07:07.369Z 2 040-059 184 184 184 301 301 301 F F F
216342024-02-15T03:07:07.369Z 3 060-079 192 192 192 323 323 323 F F F
216352024-02-15T03:07:07.369Z 4 080-099 192 194 194 323 325 325 F F F <---
216362024-02-15T03:07:07.369Z 5 100-119 192 192 192 323 323 323 F F F
216372024-02-15T03:07:07.369Z 6 120-139 192 192 192 323 323 323 F F F
216382024-02-15T03:07:07.369Z 7 140-159 192 194 194 323 325 325 F F F <---
216392024-02-15T03:07:07.369Z 8 160-179 192 192 192 323 323 323 F F F
216402024-02-15T03:07:07.369Z 9 180-199 189 189 189 317 317 317 F F F
216412024-02-15T03:07:07.369Z 10 200-219 187 194 194 312 325 325 F F F <---
216422024-02-15T03:07:07.369Z 11 220-239 184 194 194 302 325 325 F F F <---
216432024-02-15T03:07:07.369Z 12 240-259 190 194 194 320 325 325 F F F <---
216442024-02-15T03:07:07.369Z 13 260-279 190 194 194 320 325 325 F F F <---
216452024-02-15T03:07:07.369Z 14 280-299 192 192 192 323 323 323 F F F
216462024-02-15T03:07:07.369Z 15 300-319 192 192 192 323 323 323 F F F
216472024-02-15T03:07:07.369Z 16 320-339 192 194 194 323 325 325 F T T <---
216482024-02-15T03:07:07.369Z 17 340-359 192 194 194 323 325 325 F F F <---
216492024-02-15T03:07:07.369Z 18 360-379 190 190 190 321 321 321 F F F
216502024-02-15T03:07:07.370Z 19 380-399 192 194 194 323 324 324 F F F <---
216512024-02-15T03:07:07.370Z 20 400-419 180 180 180 298 298 298 F F F
216522024-02-15T03:07:07.370Z 21 420-439 192 192 192 323 323 323 F F F
216532024-02-15T03:07:07.370Z 22 440-459 192 192 192 323 323 323 F F F
216542024-02-15T03:07:07.370Z 23 460-479 192 192 192 323 323 323 F F F
216552024-02-15T03:07:07.370Z 24 480-499 191 194 194 322 325 325 F F F <---
216562024-02-15T03:07:07.370Z 25 500-519 192 192 192 323 323 323 F F F
216572024-02-15T03:07:07.370Z 26 520-539 188 188 188 313 313 313 F F F
216582024-02-15T03:07:07.370Z 27 540-559 191 191 191 322 322 322 F F F
216592024-02-15T03:07:07.370Z 28 560-579 192 194 194 323 324 324 T F F <---
216602024-02-15T03:07:07.370Z 29 580-599 187 194 194 312 324 324 F F F <---
216612024-02-15T03:07:07.370ZMax gen: 194, Max flush: 325
216622024-02-15T03:07:07.370ZError: Difference in extent metadata found!
216632024-02-15T03:07:07.376ZA Difference in extent metadata is expected here
216642024-02-15T03:07:07.376ZOn loop 97
216652024-02-15T03:07:07.376Z
216662024-02-15T03:07:07.376ZVerifying data now
216672024-02-15T03:07:11.999ZNo change after verify
216682024-02-15T03:07:11.999ZLoop: 97 Downstairs dump after verify (and repair):
216692024-02-15T03:07:12.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
216702024-02-15T03:07:12.027Z 0 000-019 192 192 192 323 323 323 F F F
216712024-02-15T03:07:12.027Z 1 020-039 191 191 191 322 322 322 F F F
216722024-02-15T03:07:12.028Z 2 040-059 184 184 184 301 301 301 F F F
216732024-02-15T03:07:12.028Z 3 060-079 192 192 192 323 323 323 F F F
216742024-02-15T03:07:12.028Z 4 080-099 194 194 194 325 325 325 F F F
216752024-02-15T03:07:12.028Z 5 100-119 192 192 192 323 323 323 F F F
216762024-02-15T03:07:12.028Z 6 120-139 192 192 192 323 323 323 F F F
216772024-02-15T03:07:12.028Z 7 140-159 194 194 194 325 325 325 F F F
216782024-02-15T03:07:12.028Z 8 160-179 192 192 192 323 323 323 F F F
216792024-02-15T03:07:12.028Z 9 180-199 189 189 189 317 317 317 F F F
216802024-02-15T03:07:12.028Z 10 200-219 194 194 194 325 325 325 F F F
216812024-02-15T03:07:12.028Z 11 220-239 194 194 194 325 325 325 F F F
216822024-02-15T03:07:12.028Z 12 240-259 194 194 194 325 325 325 F F F
216832024-02-15T03:07:12.029Z 13 260-279 194 194 194 325 325 325 F F F
216842024-02-15T03:07:12.029Z 14 280-299 192 192 192 323 323 323 F F F
216852024-02-15T03:07:12.029Z 15 300-319 192 192 192 323 323 323 F F F
216862024-02-15T03:07:12.029Z 16 320-339 195 195 195 326 326 326 F F F
216872024-02-15T03:07:12.029Z 17 340-359 194 194 194 325 325 325 F F F
216882024-02-15T03:07:12.029Z 18 360-379 190 190 190 321 321 321 F F F
216892024-02-15T03:07:12.029Z 19 380-399 194 194 194 324 324 324 F F F
216902024-02-15T03:07:12.029Z 20 400-419 180 180 180 298 298 298 F F F
216912024-02-15T03:07:12.029Z 21 420-439 192 192 192 323 323 323 F F F
216922024-02-15T03:07:12.029Z 22 440-459 192 192 192 323 323 323 F F F
216932024-02-15T03:07:12.029Z 23 460-479 192 192 192 323 323 323 F F F
216942024-02-15T03:07:12.029Z 24 480-499 194 194 194 325 325 325 F F F
216952024-02-15T03:07:12.029Z 25 500-519 192 192 192 323 323 323 F F F
216962024-02-15T03:07:12.030Z 26 520-539 188 188 188 313 313 313 F F F
216972024-02-15T03:07:12.030Z 27 540-559 191 191 191 322 322 322 F F F
216982024-02-15T03:07:12.030Z 28 560-579 194 194 194 324 324 324 F F F
216992024-02-15T03:07:12.030Z 29 580-599 194 194 194 324 324 324 F F F
217002024-02-15T03:07:12.030ZMax gen: 195, Max flush: 326
217012024-02-15T03:07:12.030Z
217022024-02-15T03:07:12.030ZBegin loop 98
217032024-02-15T03:07:12.031ZDownstairs to restart: 1
217042024-02-15T03:07:12.051Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.892714046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931}
217052024-02-15T03:07:12.051Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893049784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pidCrucible runtime is spawned
217062024-02-15T03:07:12.051Z":1931}
217072024-02-15T03:07:12.051Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893101717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931}
217082024-02-15T03:07:12.051Z{"msg":"Crucible 6f21489f-e99b-4e6a-be19-a3ad4b417a78 has session id: e95bba37-e37d-4b5b-84a8-41d2e95e6357","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893145792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217092024-02-15T03:07:12.051Z{"msg":"Upstairs opts: Upstairs UUID: 6f21489f-e99b-4e6a-be19-a3ad4b417a78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893203794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217102024-02-15T03:07:12.051Z{"msg":"Crucible stats registered with UUID: 6f21489f-e99b-4e6a-be19-a3ad4b417a78","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893244759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217112024-02-15T03:07:12.051Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893279055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931}
217122024-02-15T03:07:12.051Z{"msg":"6f21489f-e99b-4e6a-be19-a3ad4b417a78 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893343767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217132024-02-15T03:07:12.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893389681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217142024-02-15T03:07:12.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893419687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217152024-02-15T03:07:12.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893441635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217162024-02-15T03:07:12.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893497288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217172024-02-15T03:07:12.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893525174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217182024-02-15T03:07:12.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893550991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217192024-02-15T03:07:12.052Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893575378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217202024-02-15T03:07:12.052Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893624982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217212024-02-15T03:07:12.052Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893659008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217222024-02-15T03:07:12.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893781302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217232024-02-15T03:07:12.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.893847414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217242024-02-15T03:07:12.053Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:10.894004514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217252024-02-15T03:07:12.053Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:10.894057978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217262024-02-15T03:07:12.053Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:10.894105701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217272024-02-15T03:07:12.053Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.894145016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217282024-02-15T03:07:12.053Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.894181882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217292024-02-15T03:07:12.053Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.894228896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217302024-02-15T03:07:12.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.894567284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217312024-02-15T03:07:12.054Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.894611438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217322024-02-15T03:07:12.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.894651763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217332024-02-15T03:07:12.054Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.894695198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217342024-02-15T03:07:12.054Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.895169768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217352024-02-15T03:07:12.054Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.89531356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217362024-02-15T03:07:12.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.895661077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217372024-02-15T03:07:12.054Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.895698012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217382024-02-15T03:07:12.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.895724389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217392024-02-15T03:07:12.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.895755875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217402024-02-15T03:07:12.054Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.895780622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217412024-02-15T03:07:12.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:10.895802949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217422024-02-15T03:07:13.024Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.895428999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217432024-02-15T03:07:13.025Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.895510109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217442024-02-15T03:07:13.037Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.896436803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217452024-02-15T03:07:13.037Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.896468748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217462024-02-15T03:07:13.037Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.89701285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217472024-02-15T03:07:13.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.89757648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217482024-02-15T03:07:13.037Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897606956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217492024-02-15T03:07:13.037Z{"msg":"[0]R flush_numbers[0..12]: [323, 322, 301, 323, 325, 323, 323, 325, 323, 317, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897631683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217502024-02-15T03:07:13.037Z{"msg":"[0]R generation[0..12]: [192, 191, 184, 192, 194, 192, 192, 194, 192, 189, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.89765407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217512024-02-15T03:07:13.037Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897686196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217522024-02-15T03:07:13.037Z{"msg":"[1]R flush_numbers[0..12]: [323, 322, 301, 323, 325, 323, 323, 325, 323, 317, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897711713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217532024-02-15T03:07:13.037Z{"msg":"[1]R generation[0..12]: [192, 191, 184, 192, 194, 192, 192, 194, 192, 189, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897741829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217542024-02-15T03:07:13.037Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897756757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217552024-02-15T03:07:13.037Z{"msg":"Wait for a query_work_queue command to finish before sending IO
217562024-02-15T03:07:13.037Z[2]R flush_numbers[0..12]: [323, 322, 301, 323, 325, 323, 323, 325, 323, 317, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897779984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217572024-02-15T03:07:13.037Z{"msg":"[2]R generation[0..12]: [192, 191, 184, 192, 194, 192, 192, 194, 192, 189, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897795012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217582024-02-15T03:07:13.037Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.89781018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217592024-02-15T03:07:13.037Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897823589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217602024-02-15T03:07:13.037Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897850655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217612024-02-15T03:07:13.037Z{"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897876122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217622024-02-15T03:07:13.038Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897902079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"":"downstairs","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217632024-02-15T03:07:13.038Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897926756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217642024-02-15T03:07:13.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897960772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"0","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217652024-02-15T03:07:13.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.897982979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"1","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217662024-02-15T03:07:13.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.898010195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"client":"2","":"downstairs","session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217672024-02-15T03:07:13.038Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.898036112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217682024-02-15T03:07:13.038Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.898068018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217692024-02-15T03:07:13.038Z{"msg":"6f21489f-e99b-4e6a-be19-a3ad4b417a78 is now active with session: e95bba37-e37d-4b5b-84a8-41d2e95e6357","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.898087236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217702024-02-15T03:07:13.038Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:11.898122071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931,"session_id":"e95bba37-e37d-4b5b-84a8-41d2e95e6357"}
217712024-02-15T03:07:13.038Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
217722024-02-15T03:07:13.038Z,"level":30,"time":"2024-02-15T03:07:11.898162426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931}
217732024-02-15T03:07:13.038ZLoading write count information from file "/tmp/test_repair_verify.data"
217742024-02-15T03:07:13.038ZRun Repair workload
217752024-02-15T03:07:13.038Z01/30 Write block 54 len 36864 data: 12 14 12 16 16 14 15 15 15
217762024-02-15T03:07:13.038Z02/30 Write block 187 len 16384 data: 19 19 23 19
217772024-02-15T03:07:13.038Z03/30 Read block 26 len 12288
217782024-02-15T03:07:13.038Z04/30 Read block 389 len 40960
217792024-02-15T03:07:13.051Z05/30 Read block 389 len 16384
217802024-02-15T03:07:13.051Z06/30 Write block 102 len 28672 data: 16 14 15 12 10 10 9
217812024-02-15T03:07:13.051Z07/30 Read block 497 len 4096
217822024-02-15T03:07:13.051Z08/30 Read block 3 len 12288
217832024-02-15T03:07:13.051Z09/30 Write block 107 len 24576 data: 11 10 9 9 11 9
217842024-02-15T03:07:13.051Z10/30 Write block 593 len 8192 data: 17 16
217852024-02-15T03:07:13.051Z11/30 Read block 329 len 32768
217862024-02-15T03:07:13.051Z12/30 Read block 90 len 24576
217872024-02-15T03:07:13.051Z13/30 Read block 489 len 12288
217882024-02-15T03:07:13.051Z14/30 Read block 34 len 32768
217892024-02-15T03:07:13.051Z15/30 Read block 163 len 20480
217902024-02-15T03:07:13.052Z16/30 Write block 579 len 24576 data: 15 18 14 19 18 18
217912024-02-15T03:07:13.052Z17/30 Read block 63 len 32768
217922024-02-15T03:07:13.052Z18/30 Read block 344 len 12288
217932024-02-15T03:07:13.052Z19/30 Write block 325 len 28672 data: 16 17 15 14 16 18 20
217942024-02-15T03:07:13.052Z20/30 Write block 249 len 40960 data: 15 15 15 17 16 19 17 19 21 20
217952024-02-15T03:07:13.052Z21/30 Read block 586 len 8192
217962024-02-15T03:07:13.060Z22/30 Write block 266 len 12288 data: 13 9 9
217972024-02-15T03:07:13.060Z23/30 Read block 357 len 4096
217982024-02-15T03:07:13.060Z24/30 Write block 7 len 16384 data: 14 16 14 19
217992024-02-15T03:07:13.060Z25/30 Read block 228 len 24576
218002024-02-15T03:07:13.060Z26/30 Write block 531 len 8192 data: 13 15
218012024-02-15T03:07:13.060Z27/30 Read block 338 len 40960
218022024-02-15T03:07:13.060Z28/30 Read block 418 len 40960
218032024-02-15T03:07:13.061Z29/30 Write block 456 len 24576 data: 10 9 10 11 16 15
218042024-02-15T03:07:13.061Z30/30 Read block 352 len 12288
218052024-02-15T03:07:13.061Z----------------------------------------------------------------
218062024-02-15T03:07:13.061Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
218072024-02-15T03:07:13.061ZStates: Active Active Active
218082024-02-15T03:07:13.061ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
218092024-02-15T03:07:13.061Z 1 Acked 1000 Write 9 Done Sent Done false
218102024-02-15T03:07:13.061Z 2 Acked 1001 Write 4 Done Sent Done false
218112024-02-15T03:07:13.061Z 3 Acked 1002 Read 3 Done Sent Done false
218122024-02-15T03:07:13.061Z 4 Acked 1003 Read 10 Done Sent Done false
218132024-02-15T03:07:13.061Z 5 Acked 1004 Read 4 Done Sent Done false
218142024-02-15T03:07:13.061Z 6 Acked 1005 Write 7 Done Sent Done false
218152024-02-15T03:07:13.061Z 7 Acked 1006 Read 1 Done Sent Done false
218162024-02-15T03:07:13.061Z 8 Acked 1007 Read 3 Done Sent Done false
218172024-02-15T03:07:13.061Z 9 Acked 1008 Write 6 Done Sent Done false
218182024-02-15T03:07:13.061Z 10 Acked 1009 Write 2 Done Sent Done false
218192024-02-15T03:07:13.061Z 11 Acked 1010 Read 8 Done Sent Done false
218202024-02-15T03:07:13.061Z 12 Acked 1011 Read 6 Done Sent Done false
218212024-02-15T03:07:13.061Z 13 Acked 1012 Read 3 Done Sent Done false
218222024-02-15T03:07:13.061Z 14 Acked 1013 Read 8 Done Sent Done false
218232024-02-15T03:07:13.061Z 15 Acked 1014 Read 5 Done Sent Done false
218242024-02-15T03:07:13.061Z 16 Acked 1015 Write 6 Done Sent Done false
218252024-02-15T03:07:13.061Z 17 Acked 1016 Read 8 Done Sent Done false
218262024-02-15T03:07:13.062Z 18 Acked 1017 Read 3 Done Sent Done false
218272024-02-15T03:07:13.062Z 19 Acked 1018 Write 7 Done Sent Done false
218282024-02-15T03:07:13.062Z 20 Acked 1019 Write 10 Done Sent Done false
218292024-02-15T03:07:13.062Z 21 Acked 1020 Read 2 Done Sent Done false
218302024-02-15T03:07:13.062Z 22 Acked 1021 Write 3 Done Sent Done false
218312024-02-15T03:07:13.062Z 23 Acked 1022 Read 1 Done Sent Done false
218322024-02-15T03:07:13.062Z 24 Acked 1023 Write 4 Done Sent Done false
218332024-02-15T03:07:13.062Z 25 Acked 1024 Read 6 Done Sent Done false
218342024-02-15T03:07:13.062Z 26 Acked 1025 Write 2 Done Sent Done false
218352024-02-15T03:07:13.062Z 27 Acked 1026 Read 10 Done Sent Done false
218362024-02-15T03:07:13.062Z 28 Acked 1027 Read 10 Done Sent Done false
218372024-02-15T03:07:13.062Z 29 Acked 1028 Write 6 Done Sent Done false
218382024-02-15T03:07:13.062Z 30 Acked 1029 Read 3 Done Sent Sent false
218392024-02-15T03:07:13.062Z STATES DS:0 DS:1 DS:2 TOTAL
218402024-02-15T03:07:13.062Z New 0 0 0 0
218412024-02-15T03:07:13.062Z Sent 0 30 1 31
218422024-02-15T03:07:13.062Z Done 30 0 29 59
218432024-02-15T03:07:13.063Z Skipped 0 0 0 0
218442024-02-15T03:07:13.063Z Error 0 0 0 0
218452024-02-15T03:07:13.063ZLast Flush: 0 0 0
218462024-02-15T03:07:13.063ZDownstairs last five completed:
218472024-02-15T03:07:13.063ZUpstairs last five completed: 30 29 28 27 26
218482024-02-15T03:07:13.063Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:11.925991509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1931}
218492024-02-15T03:07:13.063ZWrote out file "/tmp/test_repair_verify.data"
218502024-02-15T03:07:13.063Z
218512024-02-15T03:07:13.070ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
218522024-02-15T03:07:13.080ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
218532024-02-15T03:07:13.080Z 0 000-019 192 192 192 323 323 323 T F T <---
218542024-02-15T03:07:13.080Z 1 020-039 191 191 191 322 322 322 F F F
218552024-02-15T03:07:13.081Z 2 040-059 184 184 184 301 301 301 T F T <---
218562024-02-15T03:07:13.081Z 3 060-079 192 192 192 323 323 323 T F T <---
218572024-02-15T03:07:13.081Z 4 080-099 194 194 194 325 325 325 F F F
218582024-02-15T03:07:13.081Z 5 100-119 192 192 192 323 323 323 T F T <---
218592024-02-15T03:07:13.081Z 6 120-139 192 192 192 323 323 323 F F F
218602024-02-15T03:07:13.081Z 7 140-159 194 194 194 325 325 325 F F F
218612024-02-15T03:07:13.081Z 8 160-179 192 192 192 323 323 323 F F F
218622024-02-15T03:07:13.087Z 9 180-199 189 189 189 317 317 317 T F T <---
218632024-02-15T03:07:13.087Z 10 200-219 194 194 194 325 325 325 F F F
218642024-02-15T03:07:13.087Z 11 220-239 194 194 194 325 325 325 F F F
218652024-02-15T03:07:13.087Z 12 240-259 194 194 194 325 325 325 T F T <---
218662024-02-15T03:07:13.087Z 13 260-279 194 194 194 325 325 325 T F T <---
218672024-02-15T03:07:13.087Z 14 280-299 192 192 192 323 323 323 F F F
218682024-02-15T03:07:13.087Z 15 300-319 192 192 192 323 323 323 F F F
218692024-02-15T03:07:13.087Z 16 320-339 195 195 195 326 326 326 T F T <---
218702024-02-15T03:07:13.087Z 17 340-359 194 194 194 325 325 325 F F F
218712024-02-15T03:07:13.087Z 18 360-379 190 190 190 321 321 321 F F F
218722024-02-15T03:07:13.087Z 19 380-399 194 194 194 324 324 324 F F F
218732024-02-15T03:07:13.087Z 20 400-419 180 180 180 298 298 298 F F F
218742024-02-15T03:07:13.087Z 21 420-439 192 192 192 323 323 323 F F F
218752024-02-15T03:07:13.087Z 22 440-459 192 192 192 323 323 323 T F T <---
218762024-02-15T03:07:13.088Z 23 460-479 192 192 192 323 323 323 T F T <---
218772024-02-15T03:07:13.088Z 24 480-499 194 194 194 325 325 325 F F F
218782024-02-15T03:07:13.088Z 25 500-519 192 192 192 323 323 323 F F F
218792024-02-15T03:07:13.088Z 26 520-539 188 188 188 313 313 313 T F T <---
218802024-02-15T03:07:13.088Z 27 540-559 191 191 191 322 322 322 F F F
218812024-02-15T03:07:13.088Z 28 560-579 194 194 194 324 324 324 T F T <---
218822024-02-15T03:07:13.088Z 29 580-599 194 194 194 324 324 324 T F T <---
218832024-02-15T03:07:13.088ZMax gen: 195, Max flush: 326
218842024-02-15T03:07:13.088ZError: Difference in extent metadata found!
218852024-02-15T03:07:13.088ZA Difference in extent metadata is expected here
218862024-02-15T03:07:13.088ZOn loop 98
218872024-02-15T03:07:13.088Z
218882024-02-15T03:07:13.092ZVerifying data now
218892024-02-15T03:07:17.777ZNo change after verify
218902024-02-15T03:07:17.777ZLoop: 98 Downstairs dump after verify (and repair):
218912024-02-15T03:07:17.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
218922024-02-15T03:07:17.794Z 0 000-019 196 196 196 327 327 327 F F F
218932024-02-15T03:07:17.794Z 1 020-039 191 191 191 322 322 322 F F F
218942024-02-15T03:07:17.794Z 2 040-059 196 196 196 327 327 327 F F F
218952024-02-15T03:07:17.794Z 3 060-079 196 196 196 327 327 327 F F F
218962024-02-15T03:07:17.795Z 4 080-099 194 194 194 325 325 325 F F F
218972024-02-15T03:07:17.795Z 5 100-119 196 196 196 327 327 327 F F F
218982024-02-15T03:07:17.795Z 6 120-139 192 192 192 323 323 323 F F F
218992024-02-15T03:07:17.795Z 7 140-159 194 194 194 325 325 325 F F F
219002024-02-15T03:07:17.839Z 8 160-179 192 192 192 323 323 323 F F F
219012024-02-15T03:07:17.840Z 9 180-199 196 196 196 327 327 327 F F F
219022024-02-15T03:07:17.840Z 10 200-219 194 194 194 325 325 325 F F F
219032024-02-15T03:07:17.840Z 11 220-239 194 194 194 325 325 325 F F F
219042024-02-15T03:07:17.840Z 12 240-259 196 196 196 327 327 327 F F F
219052024-02-15T03:07:17.840Z 13 260-279 196 196 196 327 327 327 F F F
219062024-02-15T03:07:17.840Z 14 280-299 192 192 192 323 323 323 F F F
219072024-02-15T03:07:17.840Z 15 300-319 192 192 192 323 323 323 F F F
219082024-02-15T03:07:17.840Z 16 320-339 196 196 196 327 327 327 F F F
219092024-02-15T03:07:17.840Z 17 340-359 194 194 194 325 325 325 F F F
219102024-02-15T03:07:17.840Z 18 360-379 190 190 190 321 321 321 F F F
219112024-02-15T03:07:17.840Z 19 380-399 194 194 194 324 324 324 F F F
219122024-02-15T03:07:17.840Z 20 400-419 180 180 180 298 298 298 F F F
219132024-02-15T03:07:17.841Z 21 420-439 192 192 192 323 323 323 F F F
219142024-02-15T03:07:17.841Z 22 440-459 196 196 196 327 327 327 F F F
219152024-02-15T03:07:17.841Z 23 460-479 196 196 196 327 327 327 F F F
219162024-02-15T03:07:17.841Z 24 480-499 194 194 194 325 325 325 F F F
219172024-02-15T03:07:17.841Z 25 500-519 192 192 192 323 323 323 F F F
219182024-02-15T03:07:17.841Z 26 520-539 196 196 196 327 327 327 F F F
219192024-02-15T03:07:17.841Z 27 540-559 191 191 191 322 322 322 F F F
219202024-02-15T03:07:17.841Z 28 560-579 196 196 196 327 327 327 F F F
219212024-02-15T03:07:17.841Z 29 580-599 196 196 196 327 327 327 F F F
219222024-02-15T03:07:17.841ZMax gen: 196, Max flush: 327
219232024-02-15T03:07:17.842Z
219242024-02-15T03:07:17.842ZBegin loop 99
219252024-02-15T03:07:17.842ZDownstairs to restart: 2
219262024-02-15T03:07:17.859Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.69178103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939}
219272024-02-15T03:07:17.859ZCrucible runtime is spawned
219282024-02-15T03:07:17.859Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"56eb785b9570b1bd468a7aafe9af80e337a7fae9\",\n git_commit_timestamp: \"2024-02-15T02:33:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692232624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939}
219292024-02-15T03:07:17.859Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692277288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939}
219302024-02-15T03:07:17.859Z{"msg":"Crucible efdf99c8-3077-47f5-9e16-8997f4562660 has session id: fec9eaf2-83a7-470f-84d0-4eb57f64bbaf","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692296766Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219312024-02-15T03:07:17.859Z{"msg":"Upstairs opts: Upstairs UUID: efdf99c8-3077-47f5-9e16-8997f4562660, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692337101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219322024-02-15T03:07:17.859Z{"msg":"Crucible stats registered with UUID: efdf99c8-3077-47f5-9e16-8997f4562660","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692370077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219332024-02-15T03:07:17.859Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692392764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939}
219342024-02-15T03:07:17.859Z{"msg":"efdf99c8-3077-47f5-9e16-8997f4562660 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692426569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219352024-02-15T03:07:17.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692451816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219362024-02-15T03:07:17.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692486882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219372024-02-15T03:07:17.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692516708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219382024-02-15T03:07:17.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692562812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219392024-02-15T03:07:17.860Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692598108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219402024-02-15T03:07:17.860Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692637253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219412024-02-15T03:07:17.860Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.69266141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219422024-02-15T03:07:17.860Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692705714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219432024-02-15T03:07:17.860Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692733641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219442024-02-15T03:07:17.860Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692770017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219452024-02-15T03:07:17.860Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:16.692798633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219462024-02-15T03:07:17.860Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.69282352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219472024-02-15T03:07:17.860Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:16.692851306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219482024-02-15T03:07:17.860Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:16.692892351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219492024-02-15T03:07:17.861Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692911659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219502024-02-15T03:07:17.861Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692943905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219512024-02-15T03:07:17.861Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.692968282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219522024-02-15T03:07:17.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.693454381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219532024-02-15T03:07:17.861Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.693494966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219542024-02-15T03:07:17.861Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.693521123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219552024-02-15T03:07:17.861Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.693547029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219562024-02-15T03:07:17.861Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.693980895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219572024-02-15T03:07:17.861Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.694145694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219582024-02-15T03:07:17.861Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.694518378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219592024-02-15T03:07:17.861Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.69457344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219602024-02-15T03:07:17.861Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.694592878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219612024-02-15T03:07:17.861Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.694621294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219622024-02-15T03:07:17.862Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.694648951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219632024-02-15T03:07:17.862Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:16.694664209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219642024-02-15T03:07:18.824Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.694389976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219652024-02-15T03:07:18.825Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.694480845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219662024-02-15T03:07:18.844Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.695379462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219672024-02-15T03:07:18.844Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.695422827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219682024-02-15T03:07:18.844Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.69595505Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219692024-02-15T03:07:18.844Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696541856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219702024-02-15T03:07:18.844Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696597669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219712024-02-15T03:07:18.844Z{"msg":"[0]R flush_numbers[0..12]: [327, 322, 327, 327, 325, 327, 323, 325, 323, 327, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696642224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219722024-02-15T03:07:18.844Z{"msg":"[0]R generation[0..12]: [196, 191, 196, 196, 194, 196, 192, 194, 192, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696663531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219732024-02-15T03:07:18.844Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696690768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219742024-02-15T03:07:18.844Z{"msg":"[1]R flush_numbers[0..12]: [327, 322, 327, 327, 325, 327, 323, 325, 323, 327, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696709466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219752024-02-15T03:07:18.844Z{"msg":"[1]R generation[0..12]: [196, 191, 196, 196, 194, 196, 192, 194, 192, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696735432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219762024-02-15T03:07:18.845Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
219772024-02-15T03:07:18.845Z,"time":"2024-02-15T03:07:17.69675382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219782024-02-15T03:07:18.845Z{"msg":"[2]R flush_numbers[0..12]: [327, 322, 327, 327, 325, 327, 323, 325, 323, 327, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696785886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219792024-02-15T03:07:18.845Z{"msg":"[2]R generation[0..12]: [196, 191, 196, 196, 194, 196, 192, 194, 192, 196, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696811563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219802024-02-15T03:07:18.845Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.69682974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219812024-02-15T03:07:18.845Z{"msg":"Max found gen is 197","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696854527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219822024-02-15T03:07:18.845Z{"msg":"Generation requested: 198 >= found:197","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696872145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219832024-02-15T03:07:18.845Z{"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696896142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219842024-02-15T03:07:18.845Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.69691278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"":"downstairs","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219852024-02-15T03:07:18.845Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696937477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219862024-02-15T03:07:18.845Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696953405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"0","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219872024-02-15T03:07:18.845Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.696979552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"1","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219882024-02-15T03:07:18.845Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.697000009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"client":"2","":"downstairs","session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219892024-02-15T03:07:18.845Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.697036105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219902024-02-15T03:07:18.845Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.697053232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219912024-02-15T03:07:18.845Z{"msg":"efdf99c8-3077-47f5-9e16-8997f4562660 is now active with session: fec9eaf2-83a7-470f-84d0-4eb57f64bbaf","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.697082559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219922024-02-15T03:07:18.845Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.697112035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939,"session_id":"fec9eaf2-83a7-470f-84d0-4eb57f64bbaf"}
219932024-02-15T03:07:18.845Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-15T03:07:17.697131283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939}
219942024-02-15T03:07:18.845ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
219952024-02-15T03:07:18.845ZLoading write count information from file "/tmp/test_repair_verify.data"
219962024-02-15T03:07:18.846ZRun Repair workload
219972024-02-15T03:07:18.846Z01/30 Write block 495 len 8192 data: 13 14
219982024-02-15T03:07:18.846Z02/30 Write block 94 len 40960 data: 14 14 17 20 20 18 15 17 17 15
219992024-02-15T03:07:18.846Z03/30 Read block 147 len 20480
220002024-02-15T03:07:18.846Z04/30 Read block 157 len 24576
220012024-02-15T03:07:18.846Z05/30 Write block 14 len 16384 data: 10 11 10 9
220022024-02-15T03:07:18.846Z06/30 Flush
220032024-02-15T03:07:18.846Z07/30 Write block 380 len 36864 data: 14 15 13 14 17 17 16 17 18
220042024-02-15T03:07:18.846Z08/30 Read block 575 len 40960
220052024-02-15T03:07:18.861Z09/30 Flush
220062024-02-15T03:07:18.861Z10/30 Write block 38 len 32768 data: 17 17 15 16 17 18 20 23
220072024-02-15T03:07:18.861Z11/30 Write block 156 len 8192 data: 20 21
220082024-02-15T03:07:18.861Z12/30 Read block 131 len 40960
220092024-02-15T03:07:18.862Z13/30 Read block 380 len 24576
220102024-02-15T03:07:18.862Z14/30 Write block 183 len 32768 data: 17 20 21 21 20 20 24 20
220112024-02-15T03:07:18.862Z15/30 Read block 564 len 4096
220122024-02-15T03:07:18.862Z16/30 Read block 171 len 12288
220132024-02-15T03:07:18.862Z17/30 Flush
220142024-02-15T03:07:18.862Z18/30 Write block 485 len 40960 data: 13 16 17 13 16 18 16 18 14 12
220152024-02-15T03:07:18.862Z19/30 Read block 476 len 36864
220162024-02-15T03:07:18.862Z20/30 Read block 520 len 28672
220172024-02-15T03:07:18.862Z21/30 Read block 442 len 4096
220182024-02-15T03:07:18.862Z22/30 Write block 23 len 12288 data: 7 10 9
220192024-02-15T03:07:18.862Z23/30 Write block 165 len 32768 data: 14 16 14 13 12 11 12 10
220202024-02-15T03:07:18.862Z24/30 Read block 535 len 36864
220212024-02-15T03:07:18.862Z25/30 Write block 548 len 20480 data: 14 11 10 10 10
220222024-02-15T03:07:18.862Z26/30 Read block 179 len 28672
220232024-02-15T03:07:18.862Z27/30 Write block 208 len 36864 data: 15 15 11 13 14 14 17 16 14
220242024-02-15T03:07:18.862Z28/30 Write block 118 len 24576 data: 10 9 10 10 9 10
220252024-02-15T03:07:18.862Z29/30 Write block 344 len 4096 data: 15
220262024-02-15T03:07:18.881Z30/30 Write block 442 len 40960 data: 17 15 13 14 15 13 13 10 9 10
220272024-02-15T03:07:18.881Z----------------------------------------------------------------
220282024-02-15T03:07:18.881Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
220292024-02-15T03:07:18.881ZStates: Active Active Active
220302024-02-15T03:07:18.881ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
220312024-02-15T03:07:18.881Z 1 Acked 1000 Write 2 Done Done Done false
220322024-02-15T03:07:18.881Z 2 Acked 1001 Write 10 Done Done Sent false
220332024-02-15T03:07:18.881Z 3 Acked 1002 Read 5 Done Done Sent false
220342024-02-15T03:07:18.881Z 4 Acked 1003 Read 6 Done Done Sent false
220352024-02-15T03:07:18.881Z 5 Acked 1004 Write 4 Done Done Sent false
220362024-02-15T03:07:18.881Z 6 Acked 1005 Flush 0 Done Done Sent false
220372024-02-15T03:07:18.881Z 7 Acked 1006 Write 9 Done Done Sent false
220382024-02-15T03:07:18.882Z 8 Acked 1007 Read 10 Done Done Sent false
220392024-02-15T03:07:18.882Z 9 Acked 1008 Flush 0 Done Done Sent false
220402024-02-15T03:07:18.882Z 10 Acked 1009 Write 8 Done Done Sent false
220412024-02-15T03:07:18.882Z 11 Acked 1010 Write 2 Done Done Sent false
220422024-02-15T03:07:18.882Z 12 Acked 1011 Read 10 Done Done Sent false
220432024-02-15T03:07:18.882Z 13 Acked 1012 Read 6 Done Done Sent false
220442024-02-15T03:07:18.882Z 14 Acked 1013 Write 8 Done Done Sent false
220452024-02-15T03:07:18.882Z 15 Acked 1014 Read 1 Done Done Sent false
220462024-02-15T03:07:18.882Z 16 Acked 1015 Read 3 Done Done Sent false
220472024-02-15T03:07:18.882Z 17 Acked 1016 Flush 0 Done Done Sent false
220482024-02-15T03:07:18.882Z 18 Acked 1017 Write 10 Done Done Sent false
220492024-02-15T03:07:18.882Z 19 Acked 1018 Read 9 Done Done Sent false
220502024-02-15T03:07:18.882Z 20 Acked 1019 Read 7 Done Done Sent false
220512024-02-15T03:07:18.883Z 21 Acked 1020 Read 1 Done Done Sent false
220522024-02-15T03:07:18.883Z 22 Acked 1021 Write 3 Done Done Sent false
220532024-02-15T03:07:18.883Z 23 Acked 1022 Write 8 Done Done Sent false
220542024-02-15T03:07:18.883Z 24 Acked 1023 Read 9 Done Done Sent false
220552024-02-15T03:07:18.883Z 25 Acked 1024 Write 5 Done Done Sent false
220562024-02-15T03:07:18.883Z 26 Acked 1025 Read 7 Done Done Sent false
220572024-02-15T03:07:18.883Z 27 Acked 1026 Write 9 Done Done Sent false
220582024-02-15T03:07:18.883Z 28 Acked 1027 Write 6 Done Done Sent false
220592024-02-15T03:07:18.884Z 29 Acked 1028 Write 1 Done Done Sent false
220602024-02-15T03:07:18.884Z 30 Acked 1029 Write 10 Sent Sent Sent false
220612024-02-15T03:07:18.884Z STATES DS:0 DS:1 DS:2 TOTAL
220622024-02-15T03:07:18.884Z New 0 0 0 0
220632024-02-15T03:07:18.884Z Sent 1 1 29 31
220642024-02-15T03:07:18.884Z Done 29 29 1 59
220652024-02-15T03:07:18.884Z Skipped 0 0 0 0
220662024-02-15T03:07:18.884Z Error 0 0 0 0
220672024-02-15T03:07:18.884ZLast Flush: 1016 1016 0
220682024-02-15T03:07:18.884ZDownstairs last five completed:
220692024-02-15T03:07:18.884ZUpstairs last five completed: 30 29 28 27 26
220702024-02-15T03:07:18.884Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-15T03:07:17.73210863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":1939}
220712024-02-15T03:07:18.884ZWrote out file "/tmp/test_repair_verify.data"
220722024-02-15T03:07:18.885Z
220732024-02-15T03:07:18.885ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
220742024-02-15T03:07:18.908ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
220752024-02-15T03:07:18.908Z 0 000-019 198 198 196 328 328 327 F F F <---
220762024-02-15T03:07:18.908Z 1 020-039 198 198 191 330 330 322 T T F <---
220772024-02-15T03:07:18.908Z 2 040-059 198 198 196 330 330 327 F F F <---
220782024-02-15T03:07:18.908Z 3 060-079 196 196 196 327 327 327 F F F
220792024-02-15T03:07:18.908Z 4 080-099 198 198 194 328 328 325 F F F <---
220802024-02-15T03:07:18.908Z 5 100-119 198 198 196 328 328 327 T T F <---
220812024-02-15T03:07:18.908Z 6 120-139 192 192 192 323 323 323 T T F <---
220822024-02-15T03:07:18.908Z 7 140-159 198 198 194 330 330 325 F F F <---
220832024-02-15T03:07:18.908Z 8 160-179 192 192 192 323 323 323 T T F <---
220842024-02-15T03:07:18.909Z 9 180-199 198 198 196 330 330 327 F F F <---
220852024-02-15T03:07:18.909Z 10 200-219 194 194 194 325 325 325 T T F <---
220862024-02-15T03:07:18.909Z 11 220-239 194 194 194 325 325 325 F F F
220872024-02-15T03:07:18.909Z 12 240-259 196 196 196 327 327 327 F F F
220882024-02-15T03:07:18.909Z 13 260-279 196 196 196 327 327 327 F F F
220892024-02-15T03:07:18.909Z 14 280-299 192 192 192 323 323 323 F F F
220902024-02-15T03:07:18.909Z 15 300-319 192 192 192 323 323 323 F F F
220912024-02-15T03:07:18.909Z 16 320-339 196 196 196 327 327 327 F F F
220922024-02-15T03:07:18.909Z 17 340-359 194 194 194 325 325 325 T T F <---
220932024-02-15T03:07:18.909Z 18 360-379 190 190 190 321 321 321 F F F
220942024-02-15T03:07:18.909Z 19 380-399 198 198 194 329 329 324 F F F <---
220952024-02-15T03:07:18.909Z 20 400-419 180 180 180 298 298 298 F F F
220962024-02-15T03:07:18.909Z 21 420-439 192 192 192 323 323 323 F F F
220972024-02-15T03:07:18.910Z 22 440-459 196 196 196 327 327 327 T T F <---
220982024-02-15T03:07:18.910Z 23 460-479 196 196 196 327 327 327 F F F
220992024-02-15T03:07:18.910Z 24 480-499 198 198 194 328 328 325 T T T <---
221002024-02-15T03:07:18.910Z 25 500-519 192 192 192 323 323 323 F F F
221012024-02-15T03:07:18.910Z 26 520-539 196 196 196 327 327 327 F F F
221022024-02-15T03:07:18.910Z 27 540-559 191 191 191 322 322 322 T T F <---
221032024-02-15T03:07:18.910Z 28 560-579 196 196 196 327 327 327 F F F
221042024-02-15T03:07:18.910Z 29 580-599 196 196 196 327 327 327 F F F
221052024-02-15T03:07:18.910ZMax gen: 198, Max flush: 330
221062024-02-15T03:07:18.910ZError: Difference in extent metadata found!
221072024-02-15T03:07:18.910ZA Difference in extent metadata is expected here
221082024-02-15T03:07:18.910ZOn loop 99
221092024-02-15T03:07:18.910Z
221102024-02-15T03:07:18.910ZVerifying data now
221112024-02-15T03:07:33.464ZNo change after verify
221122024-02-15T03:07:33.464ZLoop: 99 Downstairs dump after verify (and repair):
221132024-02-15T03:07:35.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
221142024-02-15T03:07:35.462Z 0 000-019 198 198 198 328 328 328 F F F
221152024-02-15T03:07:35.462Z 1 020-039 199 199 199 331 331 331 F F F
221162024-02-15T03:07:35.462Z 2 040-059 198 198 198 330 330 330 F F F
221172024-02-15T03:07:35.462Z 3 060-079 196 196 196 327 327 327 F F F
221182024-02-15T03:07:35.462Z 4 080-099 198 198 198 328 328 328 F F F
221192024-02-15T03:07:35.462Z 5 100-119 199 199 199 331 331 331 F F F
221202024-02-15T03:07:35.462Z 6 120-139 199 199 199 331 331 331 F F F
221212024-02-15T03:07:35.462Z 7 140-159 198 198 198 330 330 330 F F F
221222024-02-15T03:07:35.463Z 8 160-179 199 199 199 331 331 331 F F F
221232024-02-15T03:07:35.463Z 9 180-199 198 198 198 330 330 330 F F F
221242024-02-15T03:07:35.463Z 10 200-219 199 199 199 331 331 331 F F F
221252024-02-15T03:07:35.463Z 11 220-239 194 194 194 325 325 325 F F F
221262024-02-15T03:07:35.463Z 12 240-259 196 196 196 327 327 327 F F F
221272024-02-15T03:07:35.463Z 13 260-279 196 196 196 327 327 327 F F F
221282024-02-15T03:07:35.463Z 14 280-299 192 192 192 323 323 323 F F F
221292024-02-15T03:07:35.463Z 15 300-319 192 192 192 323 323 323 F F F
221302024-02-15T03:07:35.463Z 16 320-339 196 196 196 327 327 327 F F F
221312024-02-15T03:07:35.463Z 17 340-359 199 199 199 331 331 331 F F F
221322024-02-15T03:07:35.463Z 18 360-379 190 190 190 321 321 321 F F F
221332024-02-15T03:07:35.463Z 19 380-399 198 198 198 329 329 329 F F F
221342024-02-15T03:07:35.463Z 20 400-419 180 180 180 298 298 298 F F F
221352024-02-15T03:07:35.463Z 21 420-439 192 192 192 323 323 323 F F F
221362024-02-15T03:07:35.463Z 22 440-459 199 199 199 331 331 331 F F F
221372024-02-15T03:07:35.463Z 23 460-479 196 196 196 327 327 327 F F F
221382024-02-15T03:07:35.463Z 24 480-499 199 199 199 331 331 331 F F F
221392024-02-15T03:07:35.463Z 25 500-519 192 192 192 323 323 323 F F F
221402024-02-15T03:07:35.464Z 26 520-539 196 196 196 327 327 327 F F F
221412024-02-15T03:07:35.464Z 27 540-559 199 199 199 331 331 331 F F F
221422024-02-15T03:07:35.464Z 28 560-579 196 196 196 327 327 327 F F F
221432024-02-15T03:07:35.464Z 29 580-599 196 196 196 327 327 327 F F F
221442024-02-15T03:07:35.464ZMax gen: 199, Max flush: 331
221452024-02-15T03:07:35.464Z10:26 Test duration
221462024-02-15T03:07:35.464ZTest completed
221472024-02-15T03:07:35.464Z
221482024-02-15T03:07:35.464Zreal 10:26.051327257
221492024-02-15T03:07:35.464Zuser 1:40.401734341
221502024-02-15T03:07:35.464Zsys 18.658310164
221512024-02-15T03:07:35.464Ztrap 0.093633240
221522024-02-15T03:07:35.464Ztflt 0.048232743
221532024-02-15T03:07:35.464Zdflt 0.044368048
221542024-02-15T03:07:35.464Zkflt 0.005001704
221552024-02-15T03:07:35.464Zlock 8:04:35.313552064
221562024-02-15T03:07:35.464Zslp 50:29.216558516
221572024-02-15T03:07:35.464Zlat 5.788940102
221582024-02-15T03:07:35.464Zstop 0.049261112
221592024-02-15T03:07:35.464Zprocess exited: duration 635235 ms, exit code 0
 
221602024-02-15T03:07:37.487Zfound 4 output files
221612024-02-15T03:07:37.571Zuploading: /tmp/test_repair_ds8810.txt (79704 bytes)
221622024-02-15T03:07:38.696Zuploaded: /tmp/test_repair_ds8810.txt
221632024-02-15T03:07:38.697Zuploading: /tmp/test_repair_ds8820.txt (36549 bytes)
221642024-02-15T03:07:40.335Zuploaded: /tmp/test_repair_ds8820.txt
221652024-02-15T03:07:40.335Zuploading: /tmp/test_repair_ds8830.txt (23219 bytes)
221662024-02-15T03:07:42.013Zuploaded: /tmp/test_repair_ds8830.txt
221672024-02-15T03:07:42.013Zuploading: /tmp/test_repair_out.txt (99863 bytes)
221682024-02-15T03:07:43.390Zuploaded: /tmp/test_repair_out.txt