01HP06J6YGQE8RGJD4YC0Q07NZ: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HP06JMFXHC7KT27M6CV3NBTJ

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-06T22:09:46.749Zjob dependencies complete; ready to run (waiting for 16 m 24 s)
22024-02-06T22:14:15.322Zjob assigned to worker 01HP07QA4XV6D2H5ZWRDQ2G28M (queued for 4 m 28 s)
32024-02-06T22:14:15.988Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-06T22:14:27.504Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-06T22:14:27.504Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-06T22:14:37.812Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-06T22:14:37.812Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-06T22:14:51.732Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-06T22:14:51.732Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-06T22:14:58.249Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-06T22:14:58.254Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-06T22:14:58.271Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-06T22:14:58.272Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-06T22:14:58.298Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-06T22:14:58.298Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-06T22:14:58.326Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-06T22:14:58.326Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-06T22:14:58.369Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-06T22:14:58.369Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-06T22:14:58.387Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-06T22:14:58.387Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-06T22:14:58.408Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-06T22:14:58.408Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-06T22:14:58.426Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-06T22:14:58.429Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-06T22:15:15.544Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-06T22:15:15.544Zstarting task 0: "setup"
282024-02-06T22:15:15.755Z++ uname -s
292024-02-06T22:15:15.763Z+ kern=SunOS
302024-02-06T22:15:15.763Z+ case "$kern" in
312024-02-06T22:15:15.763Z+ groupadd -g 12345 build
322024-02-06T22:15:15.763Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-06T22:15:17.767Z+ zfs create -o mountpoint=/work rpool/work
342024-02-06T22:15:18.057Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-06T22:15:18.064Z+ home_fs=zfs
362024-02-06T22:15:18.064Z+ [[ zfs == autofs ]]
372024-02-06T22:15:18.064Z+ mkdir -p /home/build
382024-02-06T22:15:18.065Z+ chown build:build /home/build /work
392024-02-06T22:15:20.069Z+ chmod 0700 /home/build /work
402024-02-06T22:15:20.078Zprocess exited: duration 4542 ms, exit code 0
 
412024-02-06T22:15:20.098Zstarting task 1: "authentication"
422024-02-06T22:15:20.124Zprocess exited: duration 25 ms, exit code 0
 
432024-02-06T22:15:20.142Zstarting task 2: "build"
442024-02-06T22:15:20.151Z+ banner cores
452024-02-06T22:15:20.198Z
462024-02-06T22:15:20.198Z #### #### ##### ###### ####
472024-02-06T22:15:20.199Z # # # # # # # #
482024-02-06T22:15:20.199Z # # # # # ##### ####
492024-02-06T22:15:20.199Z # # # ##### # #
502024-02-06T22:15:20.199Z # # # # # # # # #
512024-02-06T22:15:20.199Z #### #### # # ###### ####
522024-02-06T22:15:20.199Z
532024-02-06T22:15:20.224Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-06T22:15:20.224Z+ echo 'input bins dir contains:'
552024-02-06T22:15:20.224Zinput bins dir contains:
562024-02-06T22:15:20.224Z+ ls -ltr /input/build/work/bins
572024-02-06T22:15:20.224Ztotal 760572
582024-02-06T22:15:20.224Z-rw-r--r-- 1 root root 121401710 Feb 6 22:14 crucible-downstairs.gz
592024-02-06T22:15:20.224Z-rw-r--r-- 1 root root 91080338 Feb 6 22:14 crucible-hammer.gz
602024-02-06T22:15:20.224Z-rw-r--r-- 1 root root 104713218 Feb 6 22:14 crutest.gz
612024-02-06T22:15:20.225Z-rw-r--r-- 1 root root 71941606 Feb 6 22:14 dsc.gz
622024-02-06T22:15:20.225Z+ echo 'input script dir contains:'
632024-02-06T22:15:20.225Zinput script dir contains:
642024-02-06T22:15:20.225Z+ ls -ltr /input/build/work/scripts
652024-02-06T22:15:20.225Ztotal 38
662024-02-06T22:15:20.225Z-rw-r--r-- 1 root root 3056 Feb 6 22:14 perf-downstairs-tick.d
672024-02-06T22:15:20.225Z-rw-r--r-- 1 root root 939 Feb 6 22:14 test_ds.sh
682024-02-06T22:15:20.225Z-rw-r--r-- 1 root root 3746 Feb 6 22:14 test_live_repair.sh
692024-02-06T22:15:20.225Z-rw-r--r-- 1 root root 6416 Feb 6 22:14 test_repair.sh
702024-02-06T22:15:20.225Z-rw-r--r-- 1 root root 2969 Feb 6 22:14 test_replay.sh
712024-02-06T22:15:20.225Z-rw-r--r-- 1 root root 9465 Feb 6 22:14 test_up.sh
722024-02-06T22:15:20.226Z-rw-r--r-- 1 root root 2824 Feb 6 22:14 upstairs_info.d
732024-02-06T22:15:20.226Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-06T22:15:20.226Z+ echo ' chmod input script dir contains:'
752024-02-06T22:15:20.226Z chmod input script dir contains:
762024-02-06T22:15:20.226Z+ ls -ltr /input/build/work/scripts
772024-02-06T22:15:20.226Ztotal 38
782024-02-06T22:15:20.226Z-rwxr-xr-x 1 root root 3056 Feb 6 22:14 perf-downstairs-tick.d
792024-02-06T22:15:20.226Z-rwxr-xr-x 1 root root 939 Feb 6 22:14 test_ds.sh
802024-02-06T22:15:20.226Z-rwxr-xr-x 1 root root 3746 Feb 6 22:14 test_live_repair.sh
812024-02-06T22:15:20.226Z-rwxr-xr-x 1 root root 6416 Feb 6 22:14 test_repair.sh
822024-02-06T22:15:20.226Z-rwxr-xr-x 1 root root 2969 Feb 6 22:14 test_replay.sh
832024-02-06T22:15:20.227Z-rwxr-xr-x 1 root root 9465 Feb 6 22:14 test_up.sh
842024-02-06T22:15:20.227Z-rwxr-xr-x 1 root root 2824 Feb 6 22:14 upstairs_info.d
852024-02-06T22:15:20.227Z+ banner unpack
862024-02-06T22:15:20.227Z
872024-02-06T22:15:20.227Z # # # # ##### ## #### # #
882024-02-06T22:15:20.227Z # # ## # # # # # # # # #
892024-02-06T22:15:20.227Z # # # # # # # # # # ####
902024-02-06T22:15:20.227Z # # # # # ##### ###### # # #
912024-02-06T22:15:20.227Z # # # ## # # # # # # #
922024-02-06T22:15:20.227Z #### # # # # # #### # #
932024-02-06T22:15:20.228Z
942024-02-06T22:15:20.228Z+ mkdir -p /var/tmp/bins
952024-02-06T22:15:20.228Z+ for t in "$input/bins/"*.gz
962024-02-06T22:15:20.228Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-06T22:15:20.228Z+ b=crucible-downstairs.gz
982024-02-06T22:15:20.228Z+ b=crucible-downstairs
992024-02-06T22:15:20.228Z+ gunzip
1002024-02-06T22:15:23.341Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-06T22:15:23.345Z+ for t in "$input/bins/"*.gz
1022024-02-06T22:15:23.345Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-06T22:15:23.345Z+ b=crucible-hammer.gz
1042024-02-06T22:15:23.345Z+ b=crucible-hammer
1052024-02-06T22:15:23.345Z+ gunzip
1062024-02-06T22:15:25.579Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-06T22:15:25.588Z+ for t in "$input/bins/"*.gz
1082024-02-06T22:15:25.589Z++ basename /input/build/work/bins/crutest.gz
1092024-02-06T22:15:25.589Z+ b=crutest.gz
1102024-02-06T22:15:25.589Z+ b=crutest
1112024-02-06T22:15:25.589Z+ gunzip
1122024-02-06T22:15:28.223Z+ chmod +x /var/tmp/bins/crutest
1132024-02-06T22:15:28.231Z+ for t in "$input/bins/"*.gz
1142024-02-06T22:15:28.231Z++ basename /input/build/work/bins/dsc.gz
1152024-02-06T22:15:28.231Z+ b=dsc.gz
1162024-02-06T22:15:28.231Z+ b=dsc
1172024-02-06T22:15:28.231Z+ gunzip
1182024-02-06T22:15:29.989Z+ chmod +x /var/tmp/bins/dsc
1192024-02-06T22:15:29.999Z+ export BINDIR=/var/tmp/bins
1202024-02-06T22:15:29.999Z+ BINDIR=/var/tmp/bins
1212024-02-06T22:15:29.999Z+ banner repair
1222024-02-06T22:15:29.999Z
1232024-02-06T22:15:29.999Z ##### ###### ##### ## # #####
1242024-02-06T22:15:29.999Z # # # # # # # # # #
1252024-02-06T22:15:29.999Z # # ##### # # # # # # #
1262024-02-06T22:15:29.999Z ##### # ##### ###### # #####
1272024-02-06T22:15:29.999Z # # # # # # # # #
1282024-02-06T22:15:29.999Z # # ###### # # # # # #
1292024-02-06T22:15:29.999Z
1302024-02-06T22:15:29.999Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-06T22:15:30.180ZTurn off color for downstairs dump
1322024-02-06T22:15:30.195ZCreating region directory at: /var/tmp/test_repair
1332024-02-06T22:15:30.195ZCreating dsc directory at: /tmp/dsc
1342024-02-06T22:15:30.274ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.07720643
1352024-02-06T22:15:30.725ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.043894608
1362024-02-06T22:15:30.730ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.04282435
1372024-02-06T22:15:30.730ZRegion set with 3 regions was created
1382024-02-06T22:15:30.730Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-06T22:15:30.730ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-06T22:15:30.730Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.642873572Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141}
1412024-02-06T22:15:30.730Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.643682553Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141}
1422024-02-06T22:15:30.730Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.64370242Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141}
1432024-02-06T22:15:30.730Z{"msg":"Crucible 00d466f0-f056-424b-a63b-c1e2578807db has session id: 0ca7159a-4806-4567-b5ae-796959e21c74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.643716498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1442024-02-06T22:15:30.730Z{"msg":"Upstairs opts: Upstairs UUID: 00d466f0-f056-424b-a63b-c1e2578807db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.643735056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1452024-02-06T22:15:30.731Z{"msg":"Crucible stats registered with UUID: 00d466f0-f056-424b-a63b-c1e2578807db","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.643757743Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1462024-02-06T22:15:30.731ZCrucible runtime is spawned
1472024-02-06T22:15:30.731Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644210922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141}
1482024-02-06T22:15:30.731Z{"msg":"00d466f0-f056-424b-a63b-c1e2578807db active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644254776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1492024-02-06T22:15:30.731Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644281962Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1502024-02-06T22:15:30.731Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644321417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1512024-02-06T22:15:30.731Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644358212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1522024-02-06T22:15:30.731Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644383708Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1532024-02-06T22:15:30.731Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644420314Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1542024-02-06T22:15:30.731Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644459268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1552024-02-06T22:15:30.731Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644497353Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1562024-02-06T22:15:30.731Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644525989Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1572024-02-06T22:15:30.731Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644559095Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1582024-02-06T22:15:30.731Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644770886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1592024-02-06T22:15:30.731Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644809151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1602024-02-06T22:15:30.731Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.644833587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1612024-02-06T22:15:30.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.646491914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1622024-02-06T22:15:30.731Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.64652065Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1632024-02-06T22:15:30.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.646535817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1642024-02-06T22:15:30.731Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.646566593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1652024-02-06T22:15:30.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.64658924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1662024-02-06T22:15:30.731Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.646639854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1672024-02-06T22:15:30.731Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.647579007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1682024-02-06T22:15:30.731Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.647713309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1692024-02-06T22:15:30.732Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.647772401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1702024-02-06T22:15:30.732Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648328685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1712024-02-06T22:15:30.732Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648379499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1722024-02-06T22:15:30.732Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648398596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1732024-02-06T22:15:30.732Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648412544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1742024-02-06T22:15:30.732Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648434771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1752024-02-06T22:15:30.732Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648448629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1762024-02-06T22:15:30.732Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648461288Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1772024-02-06T22:15:30.732Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648474766Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1782024-02-06T22:15:30.732Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648572802Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1792024-02-06T22:15:30.732Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648599509Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1802024-02-06T22:15:30.732Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648773265Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1812024-02-06T22:15:30.732Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648798992Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1822024-02-06T22:15:30.732Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.64881375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1832024-02-06T22:15:30.732Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648828378Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1842024-02-06T22:15:30.732Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648865383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1852024-02-06T22:15:30.732Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648880451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1862024-02-06T22:15:30.732Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648894799Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1872024-02-06T22:15:30.732Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648915446Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1882024-02-06T22:15:30.732Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648929804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1892024-02-06T22:15:30.732Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648942742Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1902024-02-06T22:15:30.732Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.64896331Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1912024-02-06T22:15:30.732Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648977218Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1922024-02-06T22:15:30.732Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.648989346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1932024-02-06T22:15:30.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.649011063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1942024-02-06T22:15:30.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.649024991Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1952024-02-06T22:15:30.733Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.649045778Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1962024-02-06T22:15:30.733Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1972024-02-06T22:15:30.733Z,"time":"2024-02-06T22:15:29.649058727Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1982024-02-06T22:15:30.733Z{"msg":"00d466f0-f056-424b-a63b-c1e2578807db is now active with session: 0ca7159a-4806-4567-b5ae-796959e21c74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.649074385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
1992024-02-06T22:15:30.733Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.649093962Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141,"session_id":"0ca7159a-4806-4567-b5ae-796959e21c74"}
2002024-02-06T22:15:30.733Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:29.64910688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141}
2012024-02-06T22:15:30.733ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-06T22:15:30.733ZFill test
2032024-02-06T22:15:30.733ZRead and Verify all blocks (0..600 range:false)
2042024-02-06T22:15:30.776ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-06T22:15:30.776ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-06T22:15:30.780Z----------------------------------------------------------------
2072024-02-06T22:15:30.780Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-06T22:15:30.780ZStates: Active Active Active
2092024-02-06T22:15:30.780ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-06T22:15:30.780Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-06T22:15:30.780Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-06T22:15:30.780Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-06T22:15:30.780Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-06T22:15:30.780Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-06T22:15:30.780Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-06T22:15:30.780Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-06T22:15:30.780Z New 0 0 0 0
2182024-02-06T22:15:30.780Z Sent 0 0 0 0
2192024-02-06T22:15:30.780Z Done 6 6 6 18
2202024-02-06T22:15:30.780Z Skipped 0 0 0 0
2212024-02-06T22:15:30.780Z Error 0 0 0 0
2222024-02-06T22:15:30.780ZLast Flush: 1006 1006 1006
2232024-02-06T22:15:30.780ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-06T22:15:30.780ZUpstairs last five completed: 13 12 11 10 9
2252024-02-06T22:15:30.781ZCLIENT: Up:0 ds:6 act:3
2262024-02-06T22:15:34.778Z----------------------------------------------------------------
2272024-02-06T22:15:34.779Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-06T22:15:34.779ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-06T22:15:34.779ZUpstairs last five completed: 14 13 12 11 10
2302024-02-06T22:15:34.779ZCLIENT: Up:0 ds:0 act:3
2312024-02-06T22:15:34.779ZCLIENT: All crucible jobs finished, exiting program
2322024-02-06T22:15:34.782Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:34.025859427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1141}
2332024-02-06T22:15:34.787Z
2342024-02-06T22:15:34.787ZBegin loop 1
2352024-02-06T22:15:34.787ZDownstairs to restart: 2
2362024-02-06T22:15:34.805Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.05055497Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143}
2372024-02-06T22:15:34.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.050840142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143}
2382024-02-06T22:15:34.819Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.050887875Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143}
2392024-02-06T22:15:34.819Z{"msg":"Crucible b1b645a8-d0bd-4c46-9b5c-37dda636e800 has session id: ff249854-75fb-40c0-a4b2-09ce27ea1042","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.050918741Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2402024-02-06T22:15:34.820Z{"msg":"Upstairs opts: Upstairs UUID: b1b645a8-d0bd-4c46-9b5c-37dda636e800, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.050977943Z","hostname":"Crucible runtime is spawned
2412024-02-06T22:15:34.820Zip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2422024-02-06T22:15:34.820Z{"msg":"Crucible stats registered with UUID: b1b645a8-d0bd-4c46-9b5c-37dda636e800","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051018208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2432024-02-06T22:15:34.820Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051066491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143}
2442024-02-06T22:15:34.820Z{"msg":"b1b645a8-d0bd-4c46-9b5c-37dda636e800 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051162258Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2452024-02-06T22:15:34.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051182095Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2462024-02-06T22:15:34.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051197133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2472024-02-06T22:15:34.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051228399Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2482024-02-06T22:15:34.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051254896Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2492024-02-06T22:15:34.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051293111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2502024-02-06T22:15:34.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051335655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2512024-02-06T22:15:34.820Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051363991Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2522024-02-06T22:15:34.820Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051409485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2532024-02-06T22:15:34.820Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051439211Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2542024-02-06T22:15:34.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051514151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2552024-02-06T22:15:34.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051556965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2562024-02-06T22:15:34.820Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:34.051770626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2572024-02-06T22:15:34.820Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:34.05181129Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2582024-02-06T22:15:34.820Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:34.051836207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2592024-02-06T22:15:34.820Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051854065Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2602024-02-06T22:15:34.820Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.05188606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2612024-02-06T22:15:34.820Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.051969049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2622024-02-06T22:15:34.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.052356867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2632024-02-06T22:15:34.821Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.052413049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2642024-02-06T22:15:34.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.052442685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2652024-02-06T22:15:34.821Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.052476351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2662024-02-06T22:15:34.821Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.053030366Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2672024-02-06T22:15:34.821Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.0530737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2682024-02-06T22:15:34.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.05359304Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2692024-02-06T22:15:34.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.053632024Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2702024-02-06T22:15:34.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.053658151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2712024-02-06T22:15:34.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.053689167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2722024-02-06T22:15:34.821Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.053713964Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2732024-02-06T22:15:34.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:34.053745719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2742024-02-06T22:15:35.815Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.053319885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2752024-02-06T22:15:35.815Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.053409042Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2762024-02-06T22:15:35.835Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.054175129Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2772024-02-06T22:15:35.835Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.054214434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2782024-02-06T22:15:35.835Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.054661133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2792024-02-06T22:15:35.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055164305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2802024-02-06T22:15:35.836Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055210609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2812024-02-06T22:15:35.836Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055232156Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2822024-02-06T22:15:35.836Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055256363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2832024-02-06T22:15:35.836Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055272351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2842024-02-06T22:15:35.836Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055293878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2852024-02-06T22:15:35.837Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055307926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2862024-02-06T22:15:35.837Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055329483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2872024-02-06T22:15:35.837Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055344421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2882024-02-06T22:15:35.837Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2892024-02-06T22:15:35.837Z,"time":"2024-02-06T22:15:35.055365588Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2902024-02-06T22:15:35.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055383346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2912024-02-06T22:15:35.838Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055398234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2922024-02-06T22:15:35.838Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.05542578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2932024-02-06T22:15:35.838Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055439688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2942024-02-06T22:15:35.838Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055453906Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2952024-02-06T22:15:35.838Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055474803Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2962024-02-06T22:15:35.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055488152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2972024-02-06T22:15:35.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055509529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2982024-02-06T22:15:35.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055523297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
2992024-02-06T22:15:35.839Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055537255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
3002024-02-06T22:15:35.839Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055557482Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
3012024-02-06T22:15:35.839Z{"msg":"b1b645a8-d0bd-4c46-9b5c-37dda636e800 is now active with session: ff249854-75fb-40c0-a4b2-09ce27ea1042","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.0555715Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
3022024-02-06T22:15:35.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055584188Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143,"session_id":"ff249854-75fb-40c0-a4b2-09ce27ea1042"}
3032024-02-06T22:15:35.840Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:35.055604636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143}
3042024-02-06T22:15:35.840ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3052024-02-06T22:15:35.840ZLoading write count information from file "/tmp/test_repair_verify.data"
3062024-02-06T22:15:35.840ZRun Repair workload
3072024-02-06T22:15:35.840Z01/30 Write block 142 len 28672 data: 2 2 2 2 2 2 2
3082024-02-06T22:15:35.840Z02/30 Read block 64 len 32768
3092024-02-06T22:15:35.841Z03/30 Write block 357 len 28672 data: 2 2 2 2 2 2 2
3102024-02-06T22:15:35.841Z04/30 Write block 398 len 20480 data: 2 2 2 2 2
3112024-02-06T22:15:35.841Z05/30 Write block 336 len 12288 data: 2 2 2
3122024-02-06T22:15:35.841Z06/30 Read block 464 len 4096
3132024-02-06T22:15:35.841Z07/30 Read block 186 len 20480
3142024-02-06T22:15:35.841Z08/30 Read block 261 len 24576
3152024-02-06T22:15:35.842Z09/30 Write block 137 len 16384 data: 2 2 2 2
3162024-02-06T22:15:35.842Z10/30 Read block 122 len 32768
3172024-02-06T22:15:35.842Z11/30 Read block 144 len 20480
3182024-02-06T22:15:35.848Z12/30 Read block 3 len 4096
3192024-02-06T22:15:35.848Z13/30 Write block 115 len 20480 data: 2 2 2 2 2
3202024-02-06T22:15:35.849Z14/30 Read block 38 len 24576
3212024-02-06T22:15:35.849Z15/30 Flush
3222024-02-06T22:15:35.857Z16/30 Write block 589 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3232024-02-06T22:15:35.862Z17/30 Read block 325 len 16384
3242024-02-06T22:15:35.862Z18/30 Read block 550 len 12288
3252024-02-06T22:15:35.862Z19/30 Read block 397 len 28672
3262024-02-06T22:15:35.862Z20/30 Flush
3272024-02-06T22:15:35.915Z21/30 Flush
3282024-02-06T22:15:35.918Z22/30 Flush
3292024-02-06T22:15:35.918Z23/30 Write block 51 len 20480 data: 2 2 2 2 2
3302024-02-06T22:15:35.918Z24/30 Read block 477 len 40960
3312024-02-06T22:15:35.918Z25/30 Read block 103 len 12288
3322024-02-06T22:15:35.921Z26/30 Read block 536 len 8192
3332024-02-06T22:15:35.921Z27/30 Write block 313 len 12288 data: 2 2 2
3342024-02-06T22:15:35.922Z28/30 Write block 261 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3352024-02-06T22:15:35.922Z29/30 Write block 565 len 4096 data: 2
3362024-02-06T22:15:35.925Z30/30 Write block 472 len 28672 data: 2 2 2 2 2 2 2
3372024-02-06T22:15:35.925Z----------------------------------------------------------------
3382024-02-06T22:15:35.925Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3392024-02-06T22:15:35.925ZStates: Active Active Active
3402024-02-06T22:15:35.925ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3412024-02-06T22:15:35.925Z 1 Acked 1000 Write 7 Done Done Done false
3422024-02-06T22:15:35.925Z 2 Acked 1001 Read 8 Done Done Done false
3432024-02-06T22:15:35.925Z 3 Acked 1002 Write 7 Done Done Done false
3442024-02-06T22:15:35.925Z 4 Acked 1003 Write 5 Done Done Sent false
3452024-02-06T22:15:35.925Z 5 Acked 1004 Write 3 Done Done Sent false
3462024-02-06T22:15:35.925Z 6 Acked 1005 Read 1 Done Done Sent false
3472024-02-06T22:15:35.926Z 7 Acked 1006 Read 5 Done Done Sent false
3482024-02-06T22:15:35.926Z 8 Acked 1007 Read 6 Done Done Sent false
3492024-02-06T22:15:35.926Z 9 Acked 1008 Write 4 Done Done Sent false
3502024-02-06T22:15:35.926Z 10 Acked 1009 Read 8 Done Done Sent false
3512024-02-06T22:15:35.926Z 11 Acked 1010 Read 5 Done Done Sent false
3522024-02-06T22:15:35.926Z 12 Acked 1011 Read 1 Done Done Sent false
3532024-02-06T22:15:35.926Z 13 Acked 1012 Write 5 Done Done Sent false
3542024-02-06T22:15:35.926Z 14 Acked 1013 Read 6 Done Done Sent false
3552024-02-06T22:15:35.926Z 15 Acked 1014 Flush 0 Done Done Sent false
3562024-02-06T22:15:35.926Z 16 Acked 1015 Write 10 Done Done Sent false
3572024-02-06T22:15:35.926Z 17 Acked 1016 Read 4 Done Done Sent false
3582024-02-06T22:15:35.926Z 18 Acked 1017 Read 3 Done Done Sent false
3592024-02-06T22:15:35.926Z 19 Acked 1018 Read 7 Done Done Sent false
3602024-02-06T22:15:35.926Z 20 Acked 1019 Flush 0 Done Done Sent false
3612024-02-06T22:15:35.926Z 21 Acked 1020 Flush 0 Done Done Sent false
3622024-02-06T22:15:35.926Z 22 Acked 1021 Flush 0 Done Done Sent false
3632024-02-06T22:15:35.926Z 23 Acked 1022 Write 5 Done Done Sent false
3642024-02-06T22:15:35.926Z 24 Acked 1023 Read 10 Done Done Sent false
3652024-02-06T22:15:35.926Z 25 Acked 1024 Read 3 Done Done Sent false
3662024-02-06T22:15:35.926Z 26 Acked 1025 Read 2 Done Done Sent false
3672024-02-06T22:15:35.926Z 27 Acked 1026 Write 3 Done Done Sent false
3682024-02-06T22:15:35.926Z 28 Acked 1027 Write 10 Done Done Sent false
3692024-02-06T22:15:35.926Z 29 Acked 1028 Write 1 Sent Sent Sent false
3702024-02-06T22:15:35.927Z 30 Acked 1029 Write 7 Sent Sent Sent false
3712024-02-06T22:15:35.927Z STATES DS:0 DS:1 DS:2 TOTAL
3722024-02-06T22:15:35.927Z New 0 0 0 0
3732024-02-06T22:15:35.927Z Sent 2 2 27 31
3742024-02-06T22:15:35.927Z Done 28 28 3 59
3752024-02-06T22:15:35.927Z Skipped 0 0 0 0
3762024-02-06T22:15:35.927Z Error 0 0 0 0
3772024-02-06T22:15:35.927ZLast Flush: 1021 1021 0
3782024-02-06T22:15:35.927ZDownstairs last five completed:
3792024-02-06T22:15:35.927ZUpstairs last five completed: 30 29 28 27 26
3802024-02-06T22:15:35.927Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:35.170114094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1143}
3812024-02-06T22:15:35.927ZWrote out file "/tmp/test_repair_verify.data"
3822024-02-06T22:15:35.931Z
3832024-02-06T22:15:35.931ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3842024-02-06T22:15:35.948ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3852024-02-06T22:15:35.948Z 0 000-019 1 1 1 1 1 1 F F F
3862024-02-06T22:15:35.957Z 1 020-039 1 1 1 1 1 1 F F F
3872024-02-06T22:15:35.957Z 2 040-059 1 1 1 1 1 1 T T F <---
3882024-02-06T22:15:35.957Z 3 060-079 1 1 1 1 1 1 F F F
3892024-02-06T22:15:35.957Z 4 080-099 1 1 1 1 1 1 F F F
3902024-02-06T22:15:35.957Z 5 100-119 2 2 1 2 2 1 F F F <---
3912024-02-06T22:15:35.957Z 6 120-139 2 2 1 2 2 1 F F F <---
3922024-02-06T22:15:35.957Z 7 140-159 2 2 1 2 2 1 F F T <---
3932024-02-06T22:15:35.957Z 8 160-179 1 1 1 1 1 1 F F F
3942024-02-06T22:15:35.957Z 9 180-199 1 1 1 1 1 1 F F F
3952024-02-06T22:15:35.957Z 10 200-219 1 1 1 1 1 1 F F F
3962024-02-06T22:15:35.957Z 11 220-239 1 1 1 1 1 1 F F F
3972024-02-06T22:15:35.957Z 12 240-259 1 1 1 1 1 1 F F F
3982024-02-06T22:15:35.957Z 13 260-279 1 1 1 1 1 1 T T F <---
3992024-02-06T22:15:35.957Z 14 280-299 1 1 1 1 1 1 F F F
4002024-02-06T22:15:35.957Z 15 300-319 1 1 1 1 1 1 T T F <---
4012024-02-06T22:15:35.957Z 16 320-339 2 2 1 2 2 1 F F F <---
4022024-02-06T22:15:35.957Z 17 340-359 2 2 1 2 2 1 F F T <---
4032024-02-06T22:15:35.958Z 18 360-379 2 2 1 2 2 1 F F T <---
4042024-02-06T22:15:35.958Z 19 380-399 2 2 1 2 2 1 F F F <---
4052024-02-06T22:15:35.958Z 20 400-419 2 2 1 2 2 1 F F F <---
4062024-02-06T22:15:35.958Z 21 420-439 1 1 1 1 1 1 F F F
4072024-02-06T22:15:35.958Z 22 440-459 1 1 1 1 1 1 F F F
4082024-02-06T22:15:35.958Z 23 460-479 1 1 1 1 1 1 T T F <---
4092024-02-06T22:15:35.958Z 24 480-499 1 1 1 1 1 1 F F F
4102024-02-06T22:15:35.958Z 25 500-519 1 1 1 1 1 1 F F F
4112024-02-06T22:15:35.958Z 26 520-539 1 1 1 1 1 1 F F F
4122024-02-06T22:15:35.958Z 27 540-559 1 1 1 1 1 1 F F F
4132024-02-06T22:15:35.958Z 28 560-579 1 1 1 1 1 1 T T F <---
4142024-02-06T22:15:35.958Z 29 580-599 2 2 1 3 3 1 F F F <---
4152024-02-06T22:15:35.958ZMax gen: 2, Max flush: 3
4162024-02-06T22:15:35.958ZError: Difference in extent metadata found!
4172024-02-06T22:15:35.958ZA Difference in extent metadata is expected here
4182024-02-06T22:15:35.958ZOn loop 1
4192024-02-06T22:15:35.958Z
4202024-02-06T22:15:35.966ZVerifying data now
4212024-02-06T22:15:43.303ZNo change after verify
4222024-02-06T22:15:43.303ZLoop: 1 Downstairs dump after verify (and repair):
4232024-02-06T22:15:43.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4242024-02-06T22:15:43.753Z 0 000-019 1 1 1 1 1 1 F F F
4252024-02-06T22:15:43.753Z 1 020-039 1 1 1 1 1 1 F F F
4262024-02-06T22:15:43.753Z 2 040-059 3 3 3 4 4 4 F F F
4272024-02-06T22:15:43.753Z 3 060-079 1 1 1 1 1 1 F F F
4282024-02-06T22:15:43.753Z 4 080-099 1 1 1 1 1 1 F F F
4292024-02-06T22:15:43.753Z 5 100-119 2 2 2 2 2 2 F F F
4302024-02-06T22:15:43.753Z 6 120-139 2 2 2 2 2 2 F F F
4312024-02-06T22:15:43.753Z 7 140-159 2 2 2 2 2 2 F F F
4322024-02-06T22:15:43.754Z 8 160-179 1 1 1 1 1 1 F F F
4332024-02-06T22:15:43.754Z 9 180-199 1 1 1 1 1 1 F F F
4342024-02-06T22:15:43.754Z 10 200-219 1 1 1 1 1 1 F F F
4352024-02-06T22:15:43.754Z 11 220-239 1 1 1 1 1 1 F F F
4362024-02-06T22:15:43.754Z 12 240-259 1 1 1 1 1 1 F F F
4372024-02-06T22:15:43.754Z 13 260-279 3 3 3 4 4 4 F F F
4382024-02-06T22:15:43.754Z 14 280-299 1 1 1 1 1 1 F F F
4392024-02-06T22:15:43.754Z 15 300-319 3 3 3 4 4 4 F F F
4402024-02-06T22:15:43.754Z 16 320-339 2 2 2 2 2 2 F F F
4412024-02-06T22:15:43.754Z 17 340-359 2 2 2 2 2 2 F F F
4422024-02-06T22:15:43.755Z 18 360-379 2 2 2 2 2 2 F F F
4432024-02-06T22:15:43.755Z 19 380-399 2 2 2 2 2 2 F F F
4442024-02-06T22:15:43.755Z 20 400-419 2 2 2 2 2 2 F F F
4452024-02-06T22:15:43.755Z 21 420-439 1 1 1 1 1 1 F F F
4462024-02-06T22:15:43.755Z 22 440-459 1 1 1 1 1 1 F F F
4472024-02-06T22:15:43.755Z 23 460-479 3 3 3 4 4 4 F F F
4482024-02-06T22:15:43.756Z 24 480-499 1 1 1 1 1 1 F F F
4492024-02-06T22:15:43.756Z 25 500-519 1 1 1 1 1 1 F F F
4502024-02-06T22:15:43.756Z 26 520-539 1 1 1 1 1 1 F F F
4512024-02-06T22:15:43.756Z 27 540-559 1 1 1 1 1 1 F F F
4522024-02-06T22:15:43.756Z 28 560-579 3 3 3 4 4 4 F F F
4532024-02-06T22:15:43.756Z 29 580-599 2 2 2 3 3 3 F F F
4542024-02-06T22:15:43.756ZMax gen: 3, Max flush: 4
4552024-02-06T22:15:43.756Z
4562024-02-06T22:15:43.756ZBegin loop 2
4572024-02-06T22:15:43.757ZDownstairs to restart: 1
4582024-02-06T22:15:43.757Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582106203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151}
4592024-02-06T22:15:43.757Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
4602024-02-06T22:15:43.757Z,"v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582450616Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151}
4612024-02-06T22:15:43.757Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.58249584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151}
4622024-02-06T22:15:43.757Z{"msg":"Crucible 511528df-23b8-40d6-83c1-aef2ed155ff5 has session id: a3404f5a-dcce-438a-a7ac-d1f7e2061400","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582528056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4632024-02-06T22:15:43.757Z{"msg":"Upstairs opts: Upstairs UUID: 511528df-23b8-40d6-83c1-aef2ed155ff5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582581329Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4642024-02-06T22:15:43.758Z{"msg":"Crucible stats registered with UUID: 511528df-23b8-40d6-83c1-aef2ed155ff5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582620943Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4652024-02-06T22:15:43.758Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582657279Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151}
4662024-02-06T22:15:43.758Z{"msg":"511528df-23b8-40d6-83c1-aef2ed155ff5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582681385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4672024-02-06T22:15:43.758Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582707282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4682024-02-06T22:15:43.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582748096Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4692024-02-06T22:15:43.758Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582808228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4702024-02-06T22:15:43.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582853482Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4712024-02-06T22:15:43.758Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582884248Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4722024-02-06T22:15:43.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582912624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4732024-02-06T22:15:43.758Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.5829401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4742024-02-06T22:15:43.758Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.582984494Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4752024-02-06T22:15:43.758Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.583008601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4762024-02-06T22:15:43.758Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.583030798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4772024-02-06T22:15:43.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.583070173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4782024-02-06T22:15:43.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.5830928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4792024-02-06T22:15:43.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.583660383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4802024-02-06T22:15:43.759Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.583701897Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4812024-02-06T22:15:43.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.583728704Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4822024-02-06T22:15:43.759Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.583770498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4832024-02-06T22:15:43.759Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584292747Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4842024-02-06T22:15:43.760Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584400163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4852024-02-06T22:15:43.760Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584605795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4862024-02-06T22:15:43.760Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584665117Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4872024-02-06T22:15:43.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584821876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4882024-02-06T22:15:43.760Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584857761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4892024-02-06T22:15:43.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584874789Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4902024-02-06T22:15:43.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584960587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4912024-02-06T22:15:43.760Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.584988883Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4922024-02-06T22:15:43.761Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585021849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4932024-02-06T22:15:43.761Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585211073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4942024-02-06T22:15:43.761Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585617328Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4952024-02-06T22:15:43.761Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585650874Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4962024-02-06T22:15:43.761Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 4, 1, 1, 2, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585666322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4972024-02-06T22:15:43.761Z{"msg":"[0]R generation[0..12]: [1, 1, 3, 1, 1, 2, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.58568093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4982024-02-06T22:15:43.761Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585702817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
4992024-02-06T22:15:43.762Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 4, 1, 1, 2, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585717345Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5002024-02-06T22:15:43.762Z{"msg":"[1]R generation[0..12]: [1, 1, 3, 1, 1, 2, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585738172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5012024-02-06T22:15:43.762Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585780397Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5022024-02-06T22:15:43.762Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 4, 1, 1, 2, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
5032024-02-06T22:15:43.762Z,"time":"2024-02-06T22:15:42.585794775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5042024-02-06T22:15:43.763Z{"msg":"[2]R generation[0..12]: [1, 1, 3, 1, 1, 2, 2, 2, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585816182Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5052024-02-06T22:15:43.763Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.58583068Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5062024-02-06T22:15:43.763Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585845148Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5072024-02-06T22:15:43.763Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585858836Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5082024-02-06T22:15:43.763Z{"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585871894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5092024-02-06T22:15:43.763Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585885662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5102024-02-06T22:15:43.763Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.58590628Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5112024-02-06T22:15:43.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585918868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5122024-02-06T22:15:43.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585932166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5132024-02-06T22:15:43.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585945424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5142024-02-06T22:15:43.764Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585958962Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5152024-02-06T22:15:43.764Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.58597809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5162024-02-06T22:15:43.764Z{"msg":"511528df-23b8-40d6-83c1-aef2ed155ff5 is now active with session: a3404f5a-dcce-438a-a7ac-d1f7e2061400","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.585991288Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5172024-02-06T22:15:43.764Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.586009815Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151,"session_id":"a3404f5a-dcce-438a-a7ac-d1f7e2061400"}
5182024-02-06T22:15:43.764Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:42.586022414Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151}
5192024-02-06T22:15:43.765ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5202024-02-06T22:15:43.765ZLoading write count information from file "/tmp/test_repair_verify.data"
5212024-02-06T22:15:43.765ZRun Repair workload
5222024-02-06T22:15:43.765Z01/30 Flush
5232024-02-06T22:15:43.789Z02/30 Write block 288 len 32768 data: 2 2 2 2 2 2 2 2
5242024-02-06T22:15:43.789Z03/30 Read block 166 len 24576
5252024-02-06T22:15:43.789Z04/30 Read block 551 len 32768
5262024-02-06T22:15:43.789Z05/30 Read block 54 len 28672
5272024-02-06T22:15:43.789Z06/30 Read block 595 len 20480
5282024-02-06T22:15:43.790Z07/30 Write block 536 len 8192 data: 2 2
5292024-02-06T22:15:43.790Z08/30 Write block 247 len 12288 data: 2 2 2
5302024-02-06T22:15:43.790Z09/30 Flush
5312024-02-06T22:15:43.790Z10/30 Write block 300 len 12288 data: 2 2 2
5322024-02-06T22:15:43.790Z11/30 Read block 38 len 8192
5332024-02-06T22:15:43.790Z12/30 Flush
5342024-02-06T22:15:43.791Z13/30 Write block 225 len 20480 data: 2 2 2 2 2
5352024-02-06T22:15:43.791Z14/30 Read block 497 len 28672
5362024-02-06T22:15:43.791Z15/30 Read block 88 len 20480
5372024-02-06T22:15:43.791Z16/30 Read block 172 len 36864
5382024-02-06T22:15:43.791Z17/30 Write block 432 len 36864 data: 2 2 2 2 2 2 2 2 2
5392024-02-06T22:15:43.791Z18/30 Flush
5402024-02-06T22:15:43.791Z19/30 Write block 68 len 12288 data: 2 2 2
5412024-02-06T22:15:43.792Z20/30 Write block 351 len 28672 data: 2 2 2 2 2 2 3
5422024-02-06T22:15:43.792Z21/30 Flush
5432024-02-06T22:15:43.792Z22/30 Write block 522 len 28672 data: 2 2 2 2 2 2 2
5442024-02-06T22:15:43.792Z23/30 Flush
5452024-02-06T22:15:43.792Z24/30 Write block 202 len 20480 data: 2 2 2 2 2
5462024-02-06T22:15:43.792Z25/30 Read block 287 len 12288
5472024-02-06T22:15:43.792Z26/30 Read block 169 len 40960
5482024-02-06T22:15:43.792Z27/30 Write block 351 len 12288 data: 3 3 3
5492024-02-06T22:15:43.792Z28/30 Read block 486 len 4096
5502024-02-06T22:15:43.793Z29/30 Write block 125 len 28672 data: 2 2 2 2 2 2 2
5512024-02-06T22:15:43.793Z30/30 Write block 467 len 20480 data: 2 2 2 2 2
5522024-02-06T22:15:43.793Z----------------------------------------------------------------
5532024-02-06T22:15:43.793Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:21
5542024-02-06T22:15:43.793ZStates: Active Active Active
5552024-02-06T22:15:43.793ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5562024-02-06T22:15:43.793Z 10 Acked 1009 Write 3 Done Sent Done false
5572024-02-06T22:15:43.793Z 11 Acked 1010 Read 2 Done Sent Done false
5582024-02-06T22:15:43.793Z 12 Acked 1011 Flush 0 Done Sent Done false
5592024-02-06T22:15:43.794Z 13 Acked 1012 Write 5 Done Sent Done false
5602024-02-06T22:15:43.794Z 14 Acked 1013 Read 7 Done Sent Done false
5612024-02-06T22:15:43.794Z 15 Acked 1014 Read 5 Done Sent Done false
5622024-02-06T22:15:43.794Z 16 Acked 1015 Read 9 Done Sent Done false
5632024-02-06T22:15:43.794Z 17 Acked 1016 Write 9 Done Sent Done false
5642024-02-06T22:15:43.794Z 18 Acked 1017 Flush 0 Done Sent Done false
5652024-02-06T22:15:43.794Z 19 Acked 1018 Write 3 Done Sent Done false
5662024-02-06T22:15:43.794Z 20 Acked 1019 Write 7 Done Sent Done false
5672024-02-06T22:15:43.795Z 21 Acked 1020 Flush 0 Done Sent Done false
5682024-02-06T22:15:43.795Z 22 Acked 1021 Write 7 Done Sent Done false
5692024-02-06T22:15:43.795Z 23 Acked 1022 Flush 0 Done Sent Done false
5702024-02-06T22:15:43.795Z 24 Acked 1023 Write 5 Done Sent Done false
5712024-02-06T22:15:43.795Z 25 Acked 1024 Read 3 Done Sent Done false
5722024-02-06T22:15:43.795Z 26 Acked 1025 Read 10 Done Sent Done false
5732024-02-06T22:15:43.795Z 27 Acked 1026 Write 3 Done Sent Done false
5742024-02-06T22:15:43.795Z 28 Acked 1027 Read 1 Done Sent Done false
5752024-02-06T22:15:43.795Z 29 Acked 1028 Write 7 Sent Sent Sent false
5762024-02-06T22:15:43.796Z 30 Acked 1029 Write 5 Sent Sent Sent false
5772024-02-06T22:15:43.796Z STATES DS:0 DS:1 DS:2 TOTAL
5782024-02-06T22:15:43.796Z New 0 0 0 0
5792024-02-06T22:15:43.796Z Sent 2 21 2 25
5802024-02-06T22:15:43.796Z Done 19 0 19 38
5812024-02-06T22:15:43.796Z Skipped 0 0 0 0
5822024-02-06T22:15:43.797Z Error 0 0 0 0
5832024-02-06T22:15:43.797ZLast Flush: 1022 1008 1022
5842024-02-06T22:15:43.797ZDownstairs last five completed: 1008 1007 1006 1005 1004
5852024-02-06T22:15:43.797ZUpstairs last five completed: 30 29 28 27 26
5862024-02-06T22:15:43.797Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:42.859607129Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1151}
5872024-02-06T22:15:43.797ZWrote out file "/tmp/test_repair_verify.data"
5882024-02-06T22:15:43.797Z
5892024-02-06T22:15:43.797ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5902024-02-06T22:15:43.798ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5912024-02-06T22:15:43.798Z 0 000-019 1 1 1 1 1 1 F F F
5922024-02-06T22:15:43.798Z 1 020-039 1 1 1 1 1 1 F F F
5932024-02-06T22:15:43.798Z 2 040-059 3 3 3 4 4 4 F F F
5942024-02-06T22:15:43.798Z 3 060-079 4 1 4 9 1 9 F F F <---
5952024-02-06T22:15:43.798Z 4 080-099 1 1 1 1 1 1 F F F
5962024-02-06T22:15:43.798Z 5 100-119 2 2 2 2 2 2 F F F
5972024-02-06T22:15:43.799Z 6 120-139 2 2 2 2 2 2 T F T <---
5982024-02-06T22:15:43.799Z 7 140-159 2 2 2 2 2 2 F F F
5992024-02-06T22:15:43.799Z 8 160-179 1 1 1 1 1 1 F F F
6002024-02-06T22:15:43.799Z 9 180-199 1 1 1 1 1 1 F F F
6012024-02-06T22:15:43.799Z 10 200-219 1 1 1 1 1 1 T F T <---
6022024-02-06T22:15:43.799Z 11 220-239 4 1 4 8 1 8 F F F <---
6032024-02-06T22:15:43.799Z 12 240-259 4 4 4 6 6 6 F F F
6042024-02-06T22:15:43.799Z 13 260-279 3 3 3 4 4 4 F F F
6052024-02-06T22:15:43.800Z 14 280-299 4 4 4 6 6 6 F F F
6062024-02-06T22:15:43.800Z 15 300-319 4 3 4 7 4 7 F F F <---
6072024-02-06T22:15:43.800Z 16 320-339 2 2 2 2 2 2 F F F
6082024-02-06T22:15:43.800Z 17 340-359 4 2 4 9 2 9 T F T <---
6092024-02-06T22:15:43.800Z 18 360-379 2 2 2 2 2 2 F F F
6102024-02-06T22:15:43.800Z 19 380-399 2 2 2 2 2 2 F F F
6112024-02-06T22:15:43.800Z 20 400-419 2 2 2 2 2 2 F F F
6122024-02-06T22:15:43.801Z 21 420-439 4 1 4 8 1 8 F F F <---
6132024-02-06T22:15:43.801Z 22 440-459 4 1 4 8 1 8 F F F <---
6142024-02-06T22:15:43.801Z 23 460-479 3 3 3 4 4 4 T F T <---
6152024-02-06T22:15:43.801Z 24 480-499 1 1 1 1 1 1 F F F
6162024-02-06T22:15:43.801Z 25 500-519 1 1 1 1 1 1 F F F
6172024-02-06T22:15:43.801Z 26 520-539 4 4 4 10 6 10 F F F <---
6182024-02-06T22:15:43.801Z 27 540-559 1 1 1 1 1 1 F F F
6192024-02-06T22:15:43.802Z 28 560-579 3 3 3 4 4 4 F F F
6202024-02-06T22:15:43.802Z 29 580-599 2 2 2 3 3 3 F F F
6212024-02-06T22:15:43.802ZMax gen: 4, Max flush: 10
6222024-02-06T22:15:43.802ZError: Difference in extent metadata found!
6232024-02-06T22:15:43.846ZA Difference in extent metadata is expected here
6242024-02-06T22:15:43.846ZOn loop 2
6252024-02-06T22:15:43.846Z
6262024-02-06T22:15:43.846ZVerifying data now
6272024-02-06T22:15:50.521ZNo change after verify
6282024-02-06T22:15:50.521ZLoop: 2 Downstairs dump after verify (and repair):
6292024-02-06T22:15:50.953ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6302024-02-06T22:15:50.953Z 0 000-019 1 1 1 1 1 1 F F F
6312024-02-06T22:15:50.953Z 1 020-039 1 1 1 1 1 1 F F F
6322024-02-06T22:15:50.954Z 2 040-059 3 3 3 4 4 4 F F F
6332024-02-06T22:15:50.954Z 3 060-079 4 4 4 9 9 9 F F F
6342024-02-06T22:15:50.954Z 4 080-099 1 1 1 1 1 1 F F F
6352024-02-06T22:15:50.954Z 5 100-119 2 2 2 2 2 2 F F F
6362024-02-06T22:15:50.954Z 6 120-139 5 5 5 11 11 11 F F F
6372024-02-06T22:15:50.966Z 7 140-159 2 2 2 2 2 2 F F F
6382024-02-06T22:15:50.966Z 8 160-179 1 1 1 1 1 1 F F F
6392024-02-06T22:15:50.966Z 9 180-199 1 1 1 1 1 1 F F F
6402024-02-06T22:15:50.966Z 10 200-219 5 5 5 11 11 11 F F F
6412024-02-06T22:15:50.966Z 11 220-239 4 4 4 8 8 8 F F F
6422024-02-06T22:15:50.966Z 12 240-259 4 4 4 6 6 6 F F F
6432024-02-06T22:15:50.966Z 13 260-279 3 3 3 4 4 4 F F F
6442024-02-06T22:15:50.967Z 14 280-299 4 4 4 6 6 6 F F F
6452024-02-06T22:15:50.967Z 15 300-319 4 4 4 7 7 7 F F F
6462024-02-06T22:15:50.967Z 16 320-339 2 2 2 2 2 2 F F F
6472024-02-06T22:15:50.967Z 17 340-359 5 5 5 11 11 11 F F F
6482024-02-06T22:15:50.967Z 18 360-379 2 2 2 2 2 2 F F F
6492024-02-06T22:15:50.967Z 19 380-399 2 2 2 2 2 2 F F F
6502024-02-06T22:15:50.967Z 20 400-419 2 2 2 2 2 2 F F F
6512024-02-06T22:15:50.967Z 21 420-439 4 4 4 8 8 8 F F F
6522024-02-06T22:15:50.967Z 22 440-459 4 4 4 8 8 8 F F F
6532024-02-06T22:15:50.967Z 23 460-479 5 5 5 11 11 11 F F F
6542024-02-06T22:15:50.967Z 24 480-499 1 1 1 1 1 1 F F F
6552024-02-06T22:15:50.967Z 25 500-519 1 1 1 1 1 1 F F F
6562024-02-06T22:15:50.967Z 26 520-539 4 4 4 10 10 10 F F F
6572024-02-06T22:15:50.967Z 27 540-559 1 1 1 1 1 1 F F F
6582024-02-06T22:15:50.967Z 28 560-579 3 3 3 4 4 4 F F F
6592024-02-06T22:15:50.967Z 29 580-599 2 2 2 3 3 3 F F F
6602024-02-06T22:15:50.967ZMax gen: 5, Max flush: 11
6612024-02-06T22:15:50.967Z
6622024-02-06T22:15:50.967ZBegin loop 3
6632024-02-06T22:15:50.967ZDownstairs to restart: 1
6642024-02-06T22:15:50.967Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.797544979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159}
6652024-02-06T22:15:50.967Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.797870585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159}
6662024-02-06T22:15:50.968Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.797891722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159}
6672024-02-06T22:15:50.968Z{"msg":"Crucible 9e8c6b3b-3f8a-4598-bd02-5d21cd2a6274 has session id: 1e1a096c-7e7d-4752-a6cb-6a4816febfc8","v":0,"name":"crucible","level":30Crucible runtime is spawned
6682024-02-06T22:15:50.968Z,"time":"2024-02-06T22:15:49.797918218Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6692024-02-06T22:15:50.968Z{"msg":"Upstairs opts: Upstairs UUID: 9e8c6b3b-3f8a-4598-bd02-5d21cd2a6274, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.797962342Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6702024-02-06T22:15:50.968Z{"msg":"Crucible stats registered with UUID: 9e8c6b3b-3f8a-4598-bd02-5d21cd2a6274","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798002527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6712024-02-06T22:15:50.968Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798018655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159}
6722024-02-06T22:15:50.968Z{"msg":"9e8c6b3b-3f8a-4598-bd02-5d21cd2a6274 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798162685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6732024-02-06T22:15:50.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798218098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6742024-02-06T22:15:50.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798248114Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6752024-02-06T22:15:50.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798268931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6762024-02-06T22:15:50.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798284379Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6772024-02-06T22:15:50.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798310485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6782024-02-06T22:15:50.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798328893Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6792024-02-06T22:15:50.968Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.79835116Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6802024-02-06T22:15:50.968Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798399783Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6812024-02-06T22:15:50.968Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798428709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6822024-02-06T22:15:50.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798453406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6832024-02-06T22:15:50.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.798478442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6842024-02-06T22:15:50.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.7986441Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6852024-02-06T22:15:50.968Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.7990887Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6862024-02-06T22:15:50.968Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.799125885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6872024-02-06T22:15:50.968Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.799244969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6882024-02-06T22:15:50.968Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.799296202Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6892024-02-06T22:15:50.968Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.799579844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6902024-02-06T22:15:50.968Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.799735903Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6912024-02-06T22:15:50.969Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.800058499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6922024-02-06T22:15:50.969Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.800099074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6932024-02-06T22:15:50.969Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.80012512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6942024-02-06T22:15:50.969Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.800215228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6952024-02-06T22:15:50.969Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.800247234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6962024-02-06T22:15:50.969Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.800298267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6972024-02-06T22:15:50.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.80034867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6982024-02-06T22:15:50.969Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.800387145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
6992024-02-06T22:15:50.969Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.800803278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7002024-02-06T22:15:50.969Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801221522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7012024-02-06T22:15:50.969Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801267296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7022024-02-06T22:15:50.969Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 4, 9, 1, 2, 11, 2, 1, 1, 11, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801303341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7032024-02-06T22:15:50.969Z{"msg":"[0]R generation[0..12]: [1, 1, 3, 4, 1, 2, 5, 2, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801339346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7042024-02-06T22:15:50.969Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801374871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7052024-02-06T22:15:50.969Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 4, 9, 1, 2, 11, 2, 1, 1, 11, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801401567Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7062024-02-06T22:15:50.969Z{"msg":"[1]R generation[0..12]: [1, 1, 3, 4, 1, 2, 5, 2, 1, 1, 5, 4]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
7072024-02-06T22:15:50.969Z,"time":"2024-02-06T22:15:49.801437253Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7082024-02-06T22:15:50.969Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801481607Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7092024-02-06T22:15:50.969Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 4, 9, 1, 2, 11, 2, 1, 1, 11, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801522281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7102024-02-06T22:15:50.969Z{"msg":"[2]R generation[0..12]: [1, 1, 3, 4, 1, 2, 5, 2, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801559906Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7112024-02-06T22:15:50.969Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801600881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7122024-02-06T22:15:50.969Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801638966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7132024-02-06T22:15:50.969Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801696008Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7142024-02-06T22:15:50.969Z{"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801732853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7152024-02-06T22:15:50.970Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801769218Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7162024-02-06T22:15:50.970Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7172024-02-06T22:15:50.970ZNo downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801809492Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7182024-02-06T22:15:50.970Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801844288Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7192024-02-06T22:15:50.970Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801861065Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7202024-02-06T22:15:50.970Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801881853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7212024-02-06T22:15:50.970Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801895791Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7222024-02-06T22:15:50.970Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801914768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7232024-02-06T22:15:50.970Z{"msg":"9e8c6b3b-3f8a-4598-bd02-5d21cd2a6274 is now active with session: 1e1a096c-7e7d-4752-a6cb-6a4816febfc8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801928546Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7242024-02-06T22:15:50.970Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801941155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159,"session_id":"1e1a096c-7e7d-4752-a6cb-6a4816febfc8"}
7252024-02-06T22:15:50.970Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:49.801953383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159}
7262024-02-06T22:15:50.970ZLoading write count information from file "/tmp/test_repair_verify.data"
7272024-02-06T22:15:50.970ZRun Repair workload
7282024-02-06T22:15:50.970Z01/30 Write block 319 len 36864 data: 2 2 2 2 2 2 2 2 2
7292024-02-06T22:15:50.970Z02/30 Read block 50 len 28672
7302024-02-06T22:15:50.970Z03/30 Read block 9 len 20480
7312024-02-06T22:15:50.970Z04/30 Write block 117 len 24576 data: 3 3 3 2 2 2
7322024-02-06T22:15:50.970Z05/30 Write block 288 len 20480 data: 3 3 3 3 3
7332024-02-06T22:15:50.970Z06/30 Write block 376 len 24576 data: 2 2 2 2 2 2
7342024-02-06T22:15:50.970Z07/30 Write block 377 len 32768 data: 3 3 3 3 3 2 2 2
7352024-02-06T22:15:50.970Z08/30 Read block 450 len 4096
7362024-02-06T22:15:50.970Z09/30 Read block 77 len 32768
7372024-02-06T22:15:50.977Z10/30 Read block 481 len 4096
7382024-02-06T22:15:50.977Z11/30 Write block 207 len 12288 data: 2 2 2
7392024-02-06T22:15:50.977Z12/30 Read block 462 len 24576
7402024-02-06T22:15:50.977Z13/30 Read block 272 len 24576
7412024-02-06T22:15:50.977Z14/30 Write block 288 len 32768 data: 4 4 4 4 4 3 3 3
7422024-02-06T22:15:50.977Z15/30 Write block 51 len 20480 data: 3 3 3 3 3
7432024-02-06T22:15:50.977Z16/30 Read block 172 len 24576
7442024-02-06T22:15:50.977Z17/30 Read block 592 len 16384
7452024-02-06T22:15:50.978Z18/30 Flush
7462024-02-06T22:15:50.978Z19/30 Write block 52 len 28672 data: 4 4 4 4 2 2 2
7472024-02-06T22:15:50.978Z20/30 Read block 401 len 36864
7482024-02-06T22:15:50.978Z21/30 Read block 127 len 32768
7492024-02-06T22:15:50.978Z22/30 Read block 131 len 8192
7502024-02-06T22:15:50.978Z23/30 Read block 42 len 12288
7512024-02-06T22:15:50.978Z24/30 Read block 432 len 16384
7522024-02-06T22:15:50.978Z25/30 Write block 242 len 36864 data: 2 2 2 2 2 3 3 3 2
7532024-02-06T22:15:50.978Z26/30 Read block 544 len 28672
7542024-02-06T22:15:50.978Z27/30 Write block 350 len 12288 data: 2 4 4
7552024-02-06T22:15:50.978Z28/30 Write block 260 len 28672 data: 2 3 3 3 3 3 3
7562024-02-06T22:15:50.978Z29/30 Read block 508 len 36864
7572024-02-06T22:15:50.978Z30/30 Write block 108 len 32768 data: 2 2 2 2 2 2 2 3
7582024-02-06T22:15:50.979Z----------------------------------------------------------------
7592024-02-06T22:15:50.979Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7602024-02-06T22:15:50.979ZStates: Active Active Active
7612024-02-06T22:15:50.979ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7622024-02-06T22:15:50.979Z 1 Acked 1000 Write 9 Done Done Done false
7632024-02-06T22:15:50.979Z 2 Acked 1001 Read 7 Done Done Done false
7642024-02-06T22:15:50.979Z 3 Acked 1002 Read 5 Done Sent Done false
7652024-02-06T22:15:50.979Z 4 Acked 1003 Write 6 Done Sent Done false
7662024-02-06T22:15:50.979Z 5 Acked 1004 Write 5 Done Sent Done false
7672024-02-06T22:15:50.979Z 6 Acked 1005 Write 6 Done Sent Done false
7682024-02-06T22:15:50.979Z 7 Acked 1006 Write 8 Done Sent Done false
7692024-02-06T22:15:50.979Z 8 Acked 1007 Read 1 Done Sent Done false
7702024-02-06T22:15:50.979Z 9 Acked 1008 Read 8 Done Sent Done false
7712024-02-06T22:15:50.980Z 10 Acked 1009 Read 1 Done Sent Done false
7722024-02-06T22:15:50.980Z 11 Acked 1010 Write 3 Done Sent Done false
7732024-02-06T22:15:50.980Z 12 Acked 1011 Read 6 Done Sent Done false
7742024-02-06T22:15:50.980Z 13 Acked 1012 Read 6 Done Sent Done false
7752024-02-06T22:15:50.980Z 14 Acked 1013 Write 8 Done Sent Done false
7762024-02-06T22:15:50.980Z 15 Acked 1014 Write 5 Done Sent Done false
7772024-02-06T22:15:50.980Z 16 Acked 1015 Read 6 Done Sent Done false
7782024-02-06T22:15:50.980Z 17 Acked 1016 Read 4 Done Sent Done false
7792024-02-06T22:15:50.980Z 18 Acked 1017 Flush 0 Done Sent Done false
7802024-02-06T22:15:50.980Z 19 Acked 1018 Write 7 Done Sent Done false
7812024-02-06T22:15:50.980Z 20 Acked 1019 Read 9 Done Sent Done false
7822024-02-06T22:15:50.980Z 21 Acked 1020 Read 8 Done Sent Done false
7832024-02-06T22:15:50.980Z 22 Acked 1021 Read 2 Done Sent Done false
7842024-02-06T22:15:50.981Z 23 Acked 1022 Read 3 Done Sent Done false
7852024-02-06T22:15:50.981Z 24 Acked 1023 Read 4 Done Sent Done false
7862024-02-06T22:15:50.981Z 25 Acked 1024 Write 9 Done Sent Done false
7872024-02-06T22:15:50.981Z 26 Acked 1025 Read 7 Done Sent Done false
7882024-02-06T22:15:50.981Z 27 Acked 1026 Write 3 Done Sent Done false
7892024-02-06T22:15:50.981Z 28 Acked 1027 Write 7 Done Sent Done false
7902024-02-06T22:15:50.981Z 29 Acked 1028 Read 9 Done Sent Done false
7912024-02-06T22:15:50.981Z 30 Acked 1029 Write 8 Sent Sent Sent false
7922024-02-06T22:15:50.981Z STATES DS:0 DS:1 DS:2 TOTAL
7932024-02-06T22:15:50.981Z New 0 0 0 0
7942024-02-06T22:15:50.981Z Sent 1 28 1 30
7952024-02-06T22:15:50.981Z Done 29 2 29 60
7962024-02-06T22:15:50.981Z Skipped 0 0 0 0
7972024-02-06T22:15:50.982Z Error 0 0 0 0
7982024-02-06T22:15:50.982ZLast Flush: 1017 0 1017
7992024-02-06T22:15:50.982ZDownstairs last five completed:
8002024-02-06T22:15:50.982ZUpstairs last five completed: 30 29 28 27 26
8012024-02-06T22:15:50.982Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:49.83143133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1159}
8022024-02-06T22:15:50.982ZWrote out file "/tmp/test_repair_verify.data"
8032024-02-06T22:15:50.982Z
8042024-02-06T22:15:50.982ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8052024-02-06T22:15:50.982ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8062024-02-06T22:15:50.982Z 0 000-019 1 1 1 1 1 1 F F F
8072024-02-06T22:15:50.982Z 1 020-039 1 1 1 1 1 1 F F F
8082024-02-06T22:15:50.982Z 2 040-059 6 3 6 12 4 12 T F T <---
8092024-02-06T22:15:50.982Z 3 060-079 4 4 4 9 9 9 F F F
8102024-02-06T22:15:50.983Z 4 080-099 1 1 1 1 1 1 F F F
8112024-02-06T22:15:50.983Z 5 100-119 6 2 6 12 2 12 T F T <---
8122024-02-06T22:15:50.983Z 6 120-139 6 5 6 12 11 12 F F F <---
8132024-02-06T22:15:50.983Z 7 140-159 2 2 2 2 2 2 F F F
8142024-02-06T22:15:50.983Z 8 160-179 1 1 1 1 1 1 F F F
8152024-02-06T22:15:50.983Z 9 180-199 1 1 1 1 1 1 F F F
8162024-02-06T22:15:50.983Z 10 200-219 6 5 6 12 11 12 F F F <---
8172024-02-06T22:15:50.983Z 11 220-239 4 4 4 8 8 8 F F F
8182024-02-06T22:15:50.983Z 12 240-259 4 4 4 6 6 6 T F T <---
8192024-02-06T22:15:50.983Z 13 260-279 3 3 3 4 4 4 T F T <---
8202024-02-06T22:15:50.983Z 14 280-299 6 4 6 12 6 12 F F F <---
8212024-02-06T22:15:50.983Z 15 300-319 6 4 6 12 7 12 F T F <---
8222024-02-06T22:15:50.983Z 16 320-339 6 2 6 12 2 12 F T F <---
8232024-02-06T22:15:50.983Z 17 340-359 5 5 5 11 11 11 T F T <---
8242024-02-06T22:15:50.984Z 18 360-379 6 2 6 12 2 12 F F F <---
8252024-02-06T22:15:50.984Z 19 380-399 6 2 6 12 2 12 F F F <---
8262024-02-06T22:15:50.984Z 20 400-419 2 2 2 2 2 2 F F F
8272024-02-06T22:15:50.984Z 21 420-439 4 4 4 8 8 8 F F F
8282024-02-06T22:15:50.984Z 22 440-459 4 4 4 8 8 8 F F F
8292024-02-06T22:15:50.984Z 23 460-479 5 5 5 11 11 11 F F F
8302024-02-06T22:15:50.984Z 24 480-499 1 1 1 1 1 1 F F F
8312024-02-06T22:15:50.984Z 25 500-519 1 1 1 1 1 1 F F F
8322024-02-06T22:15:50.984Z 26 520-539 4 4 4 10 10 10 F F F
8332024-02-06T22:15:50.984Z 27 540-559 1 1 1 1 1 1 F F F
8342024-02-06T22:15:50.984Z 28 560-579 3 3 3 4 4 4 F F F
8352024-02-06T22:15:50.984Z 29 580-599 2 2 2 3 3 3 F F F
8362024-02-06T22:15:50.984ZMax gen: 6, Max flush: 12
8372024-02-06T22:15:50.991ZError: Difference in extent metadata found!
8382024-02-06T22:15:50.991ZA Difference in extent metadata is expected here
8392024-02-06T22:15:50.991ZOn loop 3
8402024-02-06T22:15:50.991Z
8412024-02-06T22:15:50.991ZVerifying data now
8422024-02-06T22:15:56.965ZNo change after verify
8432024-02-06T22:15:56.965ZLoop: 3 Downstairs dump after verify (and repair):
8442024-02-06T22:15:56.980ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8452024-02-06T22:15:56.981Z 0 000-019 1 1 1 1 1 1 F F F
8462024-02-06T22:15:56.981Z 1 020-039 1 1 1 1 1 1 F F F
8472024-02-06T22:15:56.981Z 2 040-059 7 7 7 13 13 13 F F F
8482024-02-06T22:15:56.981Z 3 060-079 4 4 4 9 9 9 F F F
8492024-02-06T22:15:56.981Z 4 080-099 1 1 1 1 1 1 F F F
8502024-02-06T22:15:56.981Z 5 100-119 7 7 7 13 13 13 F F F
8512024-02-06T22:15:56.981Z 6 120-139 6 6 6 12 12 12 F F F
8522024-02-06T22:15:56.982Z 7 140-159 2 2 2 2 2 2 F F F
8532024-02-06T22:15:56.982Z 8 160-179 1 1 1 1 1 1 F F F
8542024-02-06T22:15:56.988Z 9 180-199 1 1 1 1 1 1 F F F
8552024-02-06T22:15:56.988Z 10 200-219 6 6 6 12 12 12 F F F
8562024-02-06T22:15:56.988Z 11 220-239 4 4 4 8 8 8 F F F
8572024-02-06T22:15:56.988Z 12 240-259 7 7 7 13 13 13 F F F
8582024-02-06T22:15:56.988Z 13 260-279 7 7 7 13 13 13 F F F
8592024-02-06T22:15:56.988Z 14 280-299 6 6 6 12 12 12 F F F
8602024-02-06T22:15:56.988Z 15 300-319 6 6 6 12 12 12 F F F
8612024-02-06T22:15:56.988Z 16 320-339 6 6 6 12 12 12 F F F
8622024-02-06T22:15:56.988Z 17 340-359 7 7 7 13 13 13 F F F
8632024-02-06T22:15:56.988Z 18 360-379 6 6 6 12 12 12 F F F
8642024-02-06T22:15:56.989Z 19 380-399 6 6 6 12 12 12 F F F
8652024-02-06T22:15:56.989Z 20 400-419 2 2 2 2 2 2 F F F
8662024-02-06T22:15:56.989Z 21 420-439 4 4 4 8 8 8 F F F
8672024-02-06T22:15:56.989Z 22 440-459 4 4 4 8 8 8 F F F
8682024-02-06T22:15:56.989Z 23 460-479 5 5 5 11 11 11 F F F
8692024-02-06T22:15:56.989Z 24 480-499 1 1 1 1 1 1 F F F
8702024-02-06T22:15:56.989Z 25 500-519 1 1 1 1 1 1 F F F
8712024-02-06T22:15:56.989Z 26 520-539 4 4 4 10 10 10 F F F
8722024-02-06T22:15:56.989Z 27 540-559 1 1 1 1 1 1 F F F
8732024-02-06T22:15:56.989Z 28 560-579 3 3 3 4 4 4 F F F
8742024-02-06T22:15:56.989Z 29 580-599 2 2 2 3 3 3 F F F
8752024-02-06T22:15:56.989ZMax gen: 7, Max flush: 13
8762024-02-06T22:15:56.989Z
8772024-02-06T22:15:56.989ZBegin loop 4
8782024-02-06T22:15:56.989ZDownstairs to restart: 1
8792024-02-06T22:15:57.003Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241397396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167}
8802024-02-06T22:15:57.003Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241627305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167}
8812024-02-06T22:15:57.003Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241652842Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167}
8822024-02-06T22:15:57.014Z{"msg":"Crucible 1beb1459-9dcf-44f0-8c96-8153b8999cc8 has session id: 67afbc3d-1206-4d69-9862-a7cbd111df8f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241685867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8832024-02-06T22:15:57.014Z{"msg":"Crucible runtime is spawned
8842024-02-06T22:15:57.014ZUpstairs opts: Upstairs UUID: 1beb1459-9dcf-44f0-8c96-8153b8999cc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241730521Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8852024-02-06T22:15:57.014Z{"msg":"Crucible stats registered with UUID: 1beb1459-9dcf-44f0-8c96-8153b8999cc8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241766786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8862024-02-06T22:15:57.014Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241792033Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167}
8872024-02-06T22:15:57.014Z{"msg":"1beb1459-9dcf-44f0-8c96-8153b8999cc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241914586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8882024-02-06T22:15:57.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241951561Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8892024-02-06T22:15:57.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241970108Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8902024-02-06T22:15:57.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.241988286Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8912024-02-06T22:15:57.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242023052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8922024-02-06T22:15:57.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242041429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8932024-02-06T22:15:57.015Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242073595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8942024-02-06T22:15:57.015Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242092712Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8952024-02-06T22:15:57.015Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242135556Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8962024-02-06T22:15:57.015Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242165952Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8972024-02-06T22:15:57.015Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:56.242295875Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8982024-02-06T22:15:57.015Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:56.242347018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
8992024-02-06T22:15:57.015Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242375624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9002024-02-06T22:15:57.015Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242415538Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9012024-02-06T22:15:57.015Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:56.242447084Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9022024-02-06T22:15:57.015Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242485099Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9032024-02-06T22:15:57.015Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242513305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9042024-02-06T22:15:57.015Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242561149Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9052024-02-06T22:15:57.015Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.242988841Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9062024-02-06T22:15:57.015Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.243031675Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9072024-02-06T22:15:57.015Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.243057622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9082024-02-06T22:15:57.015Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.243103535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9092024-02-06T22:15:57.015Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.243563233Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9102024-02-06T22:15:57.015Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.243669429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9112024-02-06T22:15:57.015Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.244068215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9122024-02-06T22:15:57.015Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.244109899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9132024-02-06T22:15:57.015Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.244149094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9142024-02-06T22:15:57.015Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.244164622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9152024-02-06T22:15:57.015Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.24417971Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9162024-02-06T22:15:57.015Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:56.244200017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9172024-02-06T22:15:58.007Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.24393938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9182024-02-06T22:15:58.038Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.244185547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9192024-02-06T22:15:58.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.244990458Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9202024-02-06T22:15:58.038Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.245046791Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9212024-02-06T22:15:58.038Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.24557201Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9222024-02-06T22:15:58.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246130354Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9232024-02-06T22:15:58.039Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246177828Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9242024-02-06T22:15:58.039Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 13, 9, 1, 13, 12, 2, 1, 1, 12, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246208074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9252024-02-06T22:15:58.039Z{"msg":"[0]R generation[0..12]: [1, 1, 7, 4, 1, 7, 6, 2, 1, 1, 6, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246224282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9262024-02-06T22:15:58.039Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246246619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9272024-02-06T22:15:58.039Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 13, 9, 1, 13, 12, 2, 1, 1, 12, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246261227Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9282024-02-06T22:15:58.039Z{"msg":"[1]R generation[0..12]: [1, 1, 7, 4, 1, 7, 6, 2, 1, 1, 6, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246275685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9292024-02-06T22:15:58.039Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246297002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9302024-02-06T22:15:58.039Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 13, 9, 1, 13, 12, 2, 1, 1, 12, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.24631166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9312024-02-06T22:15:58.040Z{"msg":"[2]R generation[0..12]: [1, 1, 7, 4, 1, 7, 6, 2, 1, 1, 6, 4]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
9322024-02-06T22:15:58.040Z2024-02-06T22:15:57.246332087Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9332024-02-06T22:15:58.040Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246350775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9342024-02-06T22:15:58.040Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246370722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9352024-02-06T22:15:58.040Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.24638502Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9362024-02-06T22:15:58.040Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246404747Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9372024-02-06T22:15:58.041Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246418405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9382024-02-06T22:15:58.041Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246448541Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9392024-02-06T22:15:58.041Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246464449Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9402024-02-06T22:15:58.041Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246491105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9412024-02-06T22:15:58.041Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246511403Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9422024-02-06T22:15:58.042Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246525301Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9432024-02-06T22:15:58.042Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246540599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9442024-02-06T22:15:58.042Z{"msg":"1beb1459-9dcf-44f0-8c96-8153b8999cc8 is now active with session: 67afbc3d-1206-4d69-9862-a7cbd111df8f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246564876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9452024-02-06T22:15:58.042Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246578074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167,"session_id":"67afbc3d-1206-4d69-9862-a7cbd111df8f"}
9462024-02-06T22:15:58.042Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:15:57.246590672Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167}
9472024-02-06T22:15:58.042ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9482024-02-06T22:15:58.043ZLoading write count information from file "/tmp/test_repair_verify.data"
9492024-02-06T22:15:58.043ZRun Repair workload
9502024-02-06T22:15:58.043Z01/30 Write block 48 len 16384 data: 2 2 2 4
9512024-02-06T22:15:58.043Z02/30 Read block 373 len 24576
9522024-02-06T22:15:58.043Z03/30 Write block 153 len 4096 data: 2
9532024-02-06T22:15:58.043Z04/30 Read block 28 len 4096
9542024-02-06T22:15:58.044Z05/30 Write block 485 len 8192 data: 2 2
9552024-02-06T22:15:58.044Z06/30 Read block 36 len 28672
9562024-02-06T22:15:58.044Z07/30 Flush
9572024-02-06T22:15:58.044Z08/30 Write block 587 len 28672 data: 2 2 3 3 3 3 3
9582024-02-06T22:15:58.044Z09/30 Read block 507 len 8192
9592024-02-06T22:15:58.044Z10/30 Read block 119 len 36864
9602024-02-06T22:15:58.044Z11/30 Flush
9612024-02-06T22:15:58.057Z12/30 Write block 80 len 24576 data: 2 2 2 2 2 2
9622024-02-06T22:15:58.058Z13/30 Write block 358 len 40960 data: 3 3 3 3 3 3 2 2 2 2
9632024-02-06T22:15:58.058Z14/30 Write block 338 len 20480 data: 3 2 2 2 2
9642024-02-06T22:15:58.058Z15/30 Read block 138 len 12288
9652024-02-06T22:15:58.058Z16/30 Flush
9662024-02-06T22:15:58.058Z17/30 Write block 580 len 4096 data: 2
9672024-02-06T22:15:58.058Z18/30 Write block 556 len 28672 data: 2 2 2 2 2 2 2
9682024-02-06T22:15:58.058Z19/30 Write block 69 len 28672 data: 3 3 2 2 2 2 2
9692024-02-06T22:15:58.059Z20/30 Write block 143 len 32768 data: 3 3 3 3 3 3 2 2
9702024-02-06T22:15:58.059Z21/30 Read block 95 len 20480
9712024-02-06T22:15:58.059Z22/30 Write block 215 len 28672 data: 2 2 2 2 2 2 2
9722024-02-06T22:15:58.059Z23/30 Read block 342 len 36864
9732024-02-06T22:15:58.059Z24/30 Flush
9742024-02-06T22:15:58.068Z25/30 Write block 145 len 24576 data: 4 4 4 4 3 3
9752024-02-06T22:15:58.069Z26/30 Write block 59 len 40960 data: 2 2 2 2 2 2 2 2 2 3
9762024-02-06T22:15:58.069Z27/30 Read block 211 len 40960
9772024-02-06T22:15:58.069Z28/30 Write block 349 len 16384 data: 2 3 5 5
9782024-02-06T22:15:58.069Z29/30 Write block 549 len 4096 data: 2
9792024-02-06T22:15:58.069Z30/30 Write block 295 len 24576 data: 4 2 2 2 2 3
9802024-02-06T22:15:58.069Z----------------------------------------------------------------
9812024-02-06T22:15:58.069Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9822024-02-06T22:15:58.070ZStates: Active Active Active
9832024-02-06T22:15:58.070ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9842024-02-06T22:15:58.070Z 1 Acked 1000 Write 4 Done Done Done false
9852024-02-06T22:15:58.070Z 2 Acked 1001 Read 6 Done Sent Done false
9862024-02-06T22:15:58.070Z 3 Acked 1002 Write 1 Done Sent Done false
9872024-02-06T22:15:58.070Z 4 Acked 1003 Read 1 Done Sent Done false
9882024-02-06T22:15:58.070Z 5 Acked 1004 Write 2 Done Sent Done false
9892024-02-06T22:15:58.071Z 6 Acked 1005 Read 7 Done Sent Done false
9902024-02-06T22:15:58.071Z 7 Acked 1006 Flush 0 Done Sent Done false
9912024-02-06T22:15:58.071Z 8 Acked 1007 Write 7 Done Sent Done false
9922024-02-06T22:15:58.071Z 9 Acked 1008 Read 2 Done Sent Done false
9932024-02-06T22:15:58.071Z 10 Acked 1009 Read 9 Done Sent Done false
9942024-02-06T22:15:58.071Z 11 Acked 1010 Flush 0 Done Sent Done false
9952024-02-06T22:15:58.071Z 12 Acked 1011 Write 6 Done Sent Done false
9962024-02-06T22:15:58.071Z 13 Acked 1012 Write 10 Done Sent Done false
9972024-02-06T22:15:58.071Z 14 Acked 1013 Write 5 Done Sent Done false
9982024-02-06T22:15:58.072Z 15 Acked 1014 Read 3 Done Sent Done false
9992024-02-06T22:15:58.072Z 16 Acked 1015 Flush 0 Done Sent Done false
10002024-02-06T22:15:58.072Z 17 Acked 1016 Write 1 Done Sent Done false
10012024-02-06T22:15:58.072Z 18 Acked 1017 Write 7 Done Sent Done false
10022024-02-06T22:15:58.072Z 19 Acked 1018 Write 7 Done Sent Done false
10032024-02-06T22:15:58.072Z 20 Acked 1019 Write 8 Done Sent Done false
10042024-02-06T22:15:58.072Z 21 Acked 1020 Read 5 Done Sent Done false
10052024-02-06T22:15:58.072Z 22 Acked 1021 Write 7 Done Sent Done false
10062024-02-06T22:15:58.073Z 23 Acked 1022 Read 9 Done Sent Done false
10072024-02-06T22:15:58.073Z 24 Acked 1023 Flush 0 Done Sent Done false
10082024-02-06T22:15:58.073Z 25 Acked 1024 Write 6 Done Sent Done false
10092024-02-06T22:15:58.073Z 26 Acked 1025 Write 10 Done Sent Done false
10102024-02-06T22:15:58.073Z 27 Acked 1026 Read 10 Done Sent Done false
10112024-02-06T22:15:58.073Z 28 Acked 1027 Write 4 Done Sent Done false
10122024-02-06T22:15:58.073Z 29 Acked 1028 Write 1 Done Sent Done false
10132024-02-06T22:15:58.074Z 30 Acked 1029 Write 6 Sent Sent Sent false
10142024-02-06T22:15:58.074Z STATES DS:0 DS:1 DS:2 TOTAL
10152024-02-06T22:15:58.074Z New 0 0 0 0
10162024-02-06T22:15:58.074Z Sent 1 29 1 31
10172024-02-06T22:15:58.074Z Done 29 1 29 59
10182024-02-06T22:15:58.074Z Skipped 0 0 0 0
10192024-02-06T22:15:58.074Z Error 0 0 0 0
10202024-02-06T22:15:58.074ZLast Flush: 1023 0 1023
10212024-02-06T22:15:58.074ZDownstairs last five completed:
10222024-02-06T22:15:58.075ZUpstairs last five completed: 30 29 28 27 26
10232024-02-06T22:15:58.075Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:15:57.307836437Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1167}
10242024-02-06T22:15:58.079ZWrote out file "/tmp/test_repair_verify.data"
10252024-02-06T22:15:58.079Z
10262024-02-06T22:15:58.079ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10272024-02-06T22:15:58.099ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10282024-02-06T22:15:58.099Z 0 000-019 1 1 1 1 1 1 F F F
10292024-02-06T22:15:58.100Z 1 020-039 1 1 1 1 1 1 F F F
10302024-02-06T22:15:58.100Z 2 040-059 8 7 8 14 13 14 T T T <---
10312024-02-06T22:15:58.100Z 3 060-079 8 4 8 17 9 17 T F T <---
10322024-02-06T22:15:58.100Z 4 080-099 8 1 8 16 1 16 F F F <---
10332024-02-06T22:15:58.100Z 5 100-119 7 7 7 13 13 13 F F F
10342024-02-06T22:15:58.100Z 6 120-139 6 6 6 12 12 12 F F F
10352024-02-06T22:15:58.100Z 7 140-159 8 2 8 17 2 17 T F T <---
10362024-02-06T22:15:58.100Z 8 160-179 1 1 1 1 1 1 F F F
10372024-02-06T22:15:58.100Z 9 180-199 1 1 1 1 1 1 F F F
10382024-02-06T22:15:58.106Z 10 200-219 8 6 8 17 12 17 F F F <---
10392024-02-06T22:15:58.106Z 11 220-239 8 4 8 17 8 17 F F F <---
10402024-02-06T22:15:58.106Z 12 240-259 7 7 7 13 13 13 F F F
10412024-02-06T22:15:58.106Z 13 260-279 7 7 7 13 13 13 F F F
10422024-02-06T22:15:58.106Z 14 280-299 6 6 6 12 12 12 T F T <---
10432024-02-06T22:15:58.106Z 15 300-319 6 6 6 12 12 12 T F T <---
10442024-02-06T22:15:58.106Z 16 320-339 8 6 8 16 12 16 F F F <---
10452024-02-06T22:15:58.106Z 17 340-359 8 7 8 16 13 16 T F T <---
10462024-02-06T22:15:58.107Z 18 360-379 8 6 8 16 12 16 F F F <---
10472024-02-06T22:15:58.107Z 19 380-399 6 6 6 12 12 12 F F F
10482024-02-06T22:15:58.107Z 20 400-419 2 2 2 2 2 2 F F F
10492024-02-06T22:15:58.107Z 21 420-439 4 4 4 8 8 8 F F F
10502024-02-06T22:15:58.107Z 22 440-459 4 4 4 8 8 8 F F F
10512024-02-06T22:15:58.107Z 23 460-479 5 5 5 11 11 11 F F F
10522024-02-06T22:15:58.108Z 24 480-499 8 1 8 14 1 14 F F F <---
10532024-02-06T22:15:58.108Z 25 500-519 1 1 1 1 1 1 F F F
10542024-02-06T22:15:58.108Z 26 520-539 4 4 4 10 10 10 F F F
10552024-02-06T22:15:58.108Z 27 540-559 8 1 8 17 1 17 T F T <---
10562024-02-06T22:15:58.108Z 28 560-579 8 3 8 17 4 17 F F F <---
10572024-02-06T22:15:58.108Z 29 580-599 8 2 8 17 3 17 F F F <---
10582024-02-06T22:15:58.109ZMax gen: 8, Max flush: 17
10592024-02-06T22:15:58.109ZError: Difference in extent metadata found!
10602024-02-06T22:15:58.109ZA Difference in extent metadata is expected here
10612024-02-06T22:15:58.109ZOn loop 4
10622024-02-06T22:15:58.109Z
10632024-02-06T22:15:58.109ZVerifying data now
10642024-02-06T22:16:04.554ZNo change after verify
10652024-02-06T22:16:04.554ZLoop: 4 Downstairs dump after verify (and repair):
10662024-02-06T22:16:04.569ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10672024-02-06T22:16:04.569Z 0 000-019 1 1 1 1 1 1 F F F
10682024-02-06T22:16:04.569Z 1 020-039 1 1 1 1 1 1 F F F
10692024-02-06T22:16:04.569Z 2 040-059 9 9 9 18 18 18 F F F
10702024-02-06T22:16:04.569Z 3 060-079 9 9 9 18 18 18 F F F
10712024-02-06T22:16:04.569Z 4 080-099 8 8 8 16 16 16 F F F
10722024-02-06T22:16:04.569Z 5 100-119 7 7 7 13 13 13 F F F
10732024-02-06T22:16:04.569Z 6 120-139 6 6 6 12 12 12 F F F
10742024-02-06T22:16:04.569Z 7 140-159 9 9 9 18 18 18 F F F
10752024-02-06T22:16:04.569Z 8 160-179 1 1 1 1 1 1 F F F
10762024-02-06T22:16:04.569Z 9 180-199 1 1 1 1 1 1 F F F
10772024-02-06T22:16:04.570Z 10 200-219 8 8 8 17 17 17 F F F
10782024-02-06T22:16:04.570Z 11 220-239 8 8 8 17 17 17 F F F
10792024-02-06T22:16:04.570Z 12 240-259 7 7 7 13 13 13 F F F
10802024-02-06T22:16:04.570Z 13 260-279 7 7 7 13 13 13 F F F
10812024-02-06T22:16:04.570Z 14 280-299 9 9 9 18 18 18 F F F
10822024-02-06T22:16:04.570Z 15 300-319 9 9 9 18 18 18 F F F
10832024-02-06T22:16:04.570Z 16 320-339 8 8 8 16 16 16 F F F
10842024-02-06T22:16:04.573Z 17 340-359 9 9 9 18 18 18 F F F
10852024-02-06T22:16:04.573Z 18 360-379 8 8 8 16 16 16 F F F
10862024-02-06T22:16:04.573Z 19 380-399 6 6 6 12 12 12 F F F
10872024-02-06T22:16:04.573Z 20 400-419 2 2 2 2 2 2 F F F
10882024-02-06T22:16:04.573Z 21 420-439 4 4 4 8 8 8 F F F
10892024-02-06T22:16:04.573Z 22 440-459 4 4 4 8 8 8 F F F
10902024-02-06T22:16:04.573Z 23 460-479 5 5 5 11 11 11 F F F
10912024-02-06T22:16:04.574Z 24 480-499 8 8 8 14 14 14 F F F
10922024-02-06T22:16:04.574Z 25 500-519 1 1 1 1 1 1 F F F
10932024-02-06T22:16:04.574Z 26 520-539 4 4 4 10 10 10 F F F
10942024-02-06T22:16:04.574Z 27 540-559 9 9 9 18 18 18 F F F
10952024-02-06T22:16:04.574Z 28 560-579 8 8 8 17 17 17 F F F
10962024-02-06T22:16:04.574Z 29 580-599 8 8 8 17 17 17 F F F
10972024-02-06T22:16:04.574ZMax gen: 9, Max flush: 18
10982024-02-06T22:16:04.574Z
10992024-02-06T22:16:04.574ZBegin loop 5
11002024-02-06T22:16:04.574ZDownstairs to restart: 0
11012024-02-06T22:16:04.589Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.830836892Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175}
11022024-02-06T22:16:04.589Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\nCrucible runtime is spawned
11032024-02-06T22:16:04.595Z git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831154599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175}
11042024-02-06T22:16:04.595Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831213141Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175}
11052024-02-06T22:16:04.595Z{"msg":"Crucible 146b3908-d475-41f1-93c3-27570f9a1854 has session id: 49ea764d-f814-4a8c-b683-51e970fc1c29","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831245966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11062024-02-06T22:16:04.595Z{"msg":"Upstairs opts: Upstairs UUID: 146b3908-d475-41f1-93c3-27570f9a1854, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831303798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11072024-02-06T22:16:04.596Z{"msg":"Crucible stats registered with UUID: 146b3908-d475-41f1-93c3-27570f9a1854","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831330205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11082024-02-06T22:16:04.596Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831344183Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175}
11092024-02-06T22:16:04.596Z{"msg":"146b3908-d475-41f1-93c3-27570f9a1854 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.83136295Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11102024-02-06T22:16:04.596Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831379488Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11112024-02-06T22:16:04.596Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831410204Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11122024-02-06T22:16:04.596Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831434371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11132024-02-06T22:16:04.596Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831462557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11142024-02-06T22:16:04.596Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831481085Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11152024-02-06T22:16:04.596Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.83151141Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11162024-02-06T22:16:04.596Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831536067Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11172024-02-06T22:16:04.596Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831596289Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11182024-02-06T22:16:04.596Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831629394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11192024-02-06T22:16:04.596Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831656621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11202024-02-06T22:16:04.596Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831682287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11212024-02-06T22:16:04.596Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.831708724Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11222024-02-06T22:16:04.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.832424357Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11232024-02-06T22:16:04.596Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.832454143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11242024-02-06T22:16:04.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.832470851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11252024-02-06T22:16:04.596Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.832485469Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11262024-02-06T22:16:04.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.832986011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11272024-02-06T22:16:04.596Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.833033515Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11282024-02-06T22:16:04.596Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.833232118Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11292024-02-06T22:16:04.596Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.833310827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11302024-02-06T22:16:04.596Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.833685987Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11312024-02-06T22:16:04.597Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.83395388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11322024-02-06T22:16:04.597Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.833991665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11332024-02-06T22:16:04.597Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834008343Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11342024-02-06T22:16:04.597Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834022221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11352024-02-06T22:16:04.597Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834038679Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11362024-02-06T22:16:04.597Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834073164Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11372024-02-06T22:16:04.597Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.83417471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11382024-02-06T22:16:04.597Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834229323Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11392024-02-06T22:16:04.597Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 18, 18, 16, 13, 12, 18, 1, 1, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834260299Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11402024-02-06T22:16:04.597Z{"msg":"[0]R generation[0..12]: [1, 1, 9, 9, 8, 7, 6, 9, 1, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834286146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11412024-02-06T22:16:04.597Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834319191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11422024-02-06T22:16:04.597Z{Wait for a query_work_queue command to finish before sending IO
11432024-02-06T22:16:04.597Z"msg":"[1]R flush_numbers[0..12]: [1, 1, 18, 18, 16, 13, 12, 18, 1, 1, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834344268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11442024-02-06T22:16:04.597Z{"msg":"[1]R generation[0..12]: [1, 1, 9, 9, 8, 7, 6, 9, 1, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834377583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11452024-02-06T22:16:04.597Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.83440322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11462024-02-06T22:16:04.597Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 18, 18, 16, 13, 12, 18, 1, 1, 17, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834436465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11472024-02-06T22:16:04.597Z{"msg":"[2]R generation[0..12]: [1, 1, 9, 9, 8, 7, 6, 9, 1, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834469831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11482024-02-06T22:16:04.597Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834502876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11492024-02-06T22:16:04.597Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834527453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11502024-02-06T22:16:04.597Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.83455097Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11512024-02-06T22:16:04.597Z{"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834582706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11522024-02-06T22:16:04.597Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834608022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11532024-02-06T22:16:04.597Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834641038Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11542024-02-06T22:16:04.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834663775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11552024-02-06T22:16:04.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.83470011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11562024-02-06T22:16:04.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834728086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11572024-02-06T22:16:04.598Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834761831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11582024-02-06T22:16:04.598Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834794427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11592024-02-06T22:16:04.598Z{"msg":"146b3908-d475-41f1-93c3-27570f9a1854 is now active with session: 49ea764d-f814-4a8c-b683-51e970fc1c29","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834820623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11602024-02-06T22:16:04.598Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834840711Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175,"session_id":"49ea764d-f814-4a8c-b683-51e970fc1c29"}
11612024-02-06T22:16:04.598Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:03.834854059Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175}
11622024-02-06T22:16:04.598ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11632024-02-06T22:16:04.598ZLoading write count information from file "/tmp/test_repair_verify.data"
11642024-02-06T22:16:04.598ZRun Repair workload
11652024-02-06T22:16:04.602Z01/30 Write block 266 len 28672 data: 4 3 3 3 3 2 2
11662024-02-06T22:16:04.602Z02/30 Flush
11672024-02-06T22:16:04.602Z03/30 Write block 14 len 24576 data: 2 2 2 2 2 2
11682024-02-06T22:16:04.602Z04/30 Write block 210 len 4096 data: 2
11692024-02-06T22:16:04.602Z05/30 Write block 417 len 8192 data: 2 2
11702024-02-06T22:16:04.602Z06/30 Read block 265 len 16384
11712024-02-06T22:16:04.602Z07/30 Flush
11722024-02-06T22:16:04.602Z08/30 Flush
11732024-02-06T22:16:04.605Z09/30 Write block 513 len 24576 data: 2 2 2 2 2 2
11742024-02-06T22:16:04.605Z10/30 Read block 250 len 12288
11752024-02-06T22:16:04.605Z11/30 Read block 548 len 8192
11762024-02-06T22:16:04.605Z12/30 Read block 359 len 28672
11772024-02-06T22:16:04.605Z13/30 Read block 320 len 20480
11782024-02-06T22:16:04.612Z14/30 Write block 564 len 4096 data: 2
11792024-02-06T22:16:04.612Z15/30 Write block 153 len 36864 data: 3 2 2 2 2 2 2 2 2
11802024-02-06T22:16:04.612Z16/30 Flush
11812024-02-06T22:16:04.636Z17/30 Write block 247 len 36864 data: 4 4 4 3 2 2 2 2 2
11822024-02-06T22:16:04.639Z18/30 Read block 263 len 8192
11832024-02-06T22:16:04.639Z19/30 Write block 121 len 32768 data: 3 3 2 2 3 3 3 3
11842024-02-06T22:16:04.639Z20/30 Write block 47 len 36864 data: 2 3 3 3 5 5 5 5 5
11852024-02-06T22:16:04.642Z21/30 Write block 502 len 8192 data: 2 2
11862024-02-06T22:16:04.642Z22/30 Write block 314 len 36864 data: 3 3 2 2 2 3 3 3 3
11872024-02-06T22:16:04.642Z23/30 Write block 350 len 40960 data: 4 6 6 4 3 3 3 4 4 4
11882024-02-06T22:16:04.642Z24/30 Write block 290 len 32768 data: 5 5 5 4 4 5 3 3
11892024-02-06T22:16:04.646Z25/30 Read block 24 len 16384
11902024-02-06T22:16:04.646Z26/30 Read block 399 len 28672
11912024-02-06T22:16:04.646Z27/30 Read block 126 len 20480
11922024-02-06T22:16:04.650Z28/30 Write block 409 len 8192 data: 2 2
11932024-02-06T22:16:04.650Z29/30 Write block 374 len 20480 data: 2 2 3 4 4
11942024-02-06T22:16:04.650Z30/30 Read block 481 len 4096
11952024-02-06T22:16:04.650Z----------------------------------------------------------------
11962024-02-06T22:16:04.650Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11972024-02-06T22:16:04.650ZStates: Active Active Active
11982024-02-06T22:16:04.650ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11992024-02-06T22:16:04.650Z 1 Acked 1000 Write 7 Sent Done Done false
12002024-02-06T22:16:04.650Z 2 Acked 1001 Flush 0 Sent Done Done false
12012024-02-06T22:16:04.650Z 3 Acked 1002 Write 6 Sent Done Done false
12022024-02-06T22:16:04.650Z 4 Acked 1003 Write 1 Sent Done Done false
12032024-02-06T22:16:04.650Z 5 Acked 1004 Write 2 Sent Done Done false
12042024-02-06T22:16:04.650Z 6 Acked 1005 Read 4 Sent Done Done false
12052024-02-06T22:16:04.650Z 7 Acked 1006 Flush 0 Sent Done Done false
12062024-02-06T22:16:04.650Z 8 Acked 1007 Flush 0 Sent Done Done false
12072024-02-06T22:16:04.650Z 9 Acked 1008 Write 6 Sent Done Done false
12082024-02-06T22:16:04.650Z 10 Acked 1009 Read 3 Sent Done Done false
12092024-02-06T22:16:04.650Z 11 Acked 1010 Read 2 Sent Done Done false
12102024-02-06T22:16:04.650Z 12 Acked 1011 Read 7 Sent Done Done false
12112024-02-06T22:16:04.650Z 13 Acked 1012 Read 5 Sent Done Done false
12122024-02-06T22:16:04.651Z 14 Acked 1013 Write 1 Sent Done Done false
12132024-02-06T22:16:04.651Z 15 Acked 1014 Write 9 Sent Done Done false
12142024-02-06T22:16:04.651Z 16 Acked 1015 Flush 0 Sent Done Done false
12152024-02-06T22:16:04.651Z 17 Acked 1016 Write 9 Sent Done Done false
12162024-02-06T22:16:04.651Z 18 Acked 1017 Read 2 Sent Done Done false
12172024-02-06T22:16:04.651Z 19 Acked 1018 Write 8 Sent Done Done false
12182024-02-06T22:16:04.651Z 20 Acked 1019 Write 9 Sent Done Done false
12192024-02-06T22:16:04.651Z 21 Acked 1020 Write 2 Sent Done Done false
12202024-02-06T22:16:04.651Z 22 Acked 1021 Write 9 Sent Done Done false
12212024-02-06T22:16:04.651Z 23 Acked 1022 Write 10 Sent Done Done false
12222024-02-06T22:16:04.651Z 24 Acked 1023 Write 8 Sent Done Done false
12232024-02-06T22:16:04.651Z 25 Acked 1024 Read 4 Sent Done Done false
12242024-02-06T22:16:04.651Z 26 Acked 1025 Read 7 Sent Done Done false
12252024-02-06T22:16:04.651Z 27 Acked 1026 Read 5 Sent Done Done false
12262024-02-06T22:16:04.651Z 28 Acked 1027 Write 2 Sent Done Done false
12272024-02-06T22:16:04.651Z 29 Acked 1028 Write 5 Sent Done Done false
12282024-02-06T22:16:04.651Z 30 Acked 1029 Read 1 Sent Done Sent false
12292024-02-06T22:16:04.651Z STATES DS:0 DS:1 DS:2 TOTAL
12302024-02-06T22:16:04.651Z New 0 0 0 0
12312024-02-06T22:16:04.651Z Sent 30 0 1 31
12322024-02-06T22:16:04.651Z Done 0 30 29 59
12332024-02-06T22:16:04.651Z Skipped 0 0 0 0
12342024-02-06T22:16:04.651Z Error 0 0 0 0
12352024-02-06T22:16:04.651ZLast Flush: 0 1015 1015
12362024-02-06T22:16:04.652ZDownstairs last five completed:
12372024-02-06T22:16:04.652ZUpstairs last five completed: 30 29 28 27 26
12382024-02-06T22:16:04.652Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:03.89065077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1175}
12392024-02-06T22:16:04.652ZWrote out file "/tmp/test_repair_verify.data"
12402024-02-06T22:16:04.655Z
12412024-02-06T22:16:04.655ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12422024-02-06T22:16:04.673ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12432024-02-06T22:16:04.673Z 0 000-019 1 10 10 1 20 20 F F F <---
12442024-02-06T22:16:04.673Z 1 020-039 1 1 1 1 1 1 F F F
12452024-02-06T22:16:04.673Z 2 040-059 9 9 9 18 18 18 F T T <---
12462024-02-06T22:16:04.673Z 3 060-079 9 9 9 18 18 18 F F F
12472024-02-06T22:16:04.673Z 4 080-099 8 8 8 16 16 16 F F F
12482024-02-06T22:16:04.676Z 5 100-119 7 7 7 13 13 13 F F F
12492024-02-06T22:16:04.677Z 6 120-139 6 6 6 12 12 12 F T T <---
12502024-02-06T22:16:04.677Z 7 140-159 9 10 10 18 22 22 F F F <---
12512024-02-06T22:16:04.677Z 8 160-179 1 10 10 1 22 22 F F F <---
12522024-02-06T22:16:04.677Z 9 180-199 1 1 1 1 1 1 F F F
12532024-02-06T22:16:04.677Z 10 200-219 8 10 10 17 20 20 F F F <---
12542024-02-06T22:16:04.677Z 11 220-239 8 8 8 17 17 17 F F F
12552024-02-06T22:16:04.677Z 12 240-259 7 7 7 13 13 13 F T T <---
12562024-02-06T22:16:04.677Z 13 260-279 7 10 10 13 19 19 F F F <---
12572024-02-06T22:16:04.677Z 14 280-299 9 9 9 18 18 18 F T T <---
12582024-02-06T22:16:04.677Z 15 300-319 9 9 9 18 18 18 F T T <---
12592024-02-06T22:16:04.677Z 16 320-339 8 8 8 16 16 16 F T T <---
12602024-02-06T22:16:04.677Z 17 340-359 9 9 9 18 18 18 F T T <---
12612024-02-06T22:16:04.677Z 18 360-379 8 8 8 16 16 16 F T T <---
12622024-02-06T22:16:04.677Z 19 380-399 6 6 6 12 12 12 F F F
12632024-02-06T22:16:04.677Z 20 400-419 2 10 10 2 20 20 F T T <---
12642024-02-06T22:16:04.677Z 21 420-439 4 4 4 8 8 8 F F F
12652024-02-06T22:16:04.677Z 22 440-459 4 4 4 8 8 8 F F F
12662024-02-06T22:16:04.677Z 23 460-479 5 5 5 11 11 11 F F F
12672024-02-06T22:16:04.677Z 24 480-499 8 8 8 14 14 14 F F F
12682024-02-06T22:16:04.677Z 25 500-519 1 10 10 1 22 22 F T T <---
12692024-02-06T22:16:04.677Z 26 520-539 4 4 4 10 10 10 F F F
12702024-02-06T22:16:04.677Z 27 540-559 9 9 9 18 18 18 F F F
12712024-02-06T22:16:04.677Z 28 560-579 8 10 10 17 22 22 F F F <---
12722024-02-06T22:16:04.678Z 29 580-599 8 8 8 17 17 17 F F F
12732024-02-06T22:16:04.678ZMax gen: 10, Max flush: 22
12742024-02-06T22:16:04.678ZError: Difference in extent metadata found!
12752024-02-06T22:16:04.678ZA Difference in extent metadata is expected here
12762024-02-06T22:16:04.678ZOn loop 5
12772024-02-06T22:16:04.678Z
12782024-02-06T22:16:04.681ZVerifying data now
12792024-02-06T22:16:09.963ZNo change after verify
12802024-02-06T22:16:09.963ZLoop: 5 Downstairs dump after verify (and repair):
12812024-02-06T22:16:09.985ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12822024-02-06T22:16:09.985Z 0 000-019 10 10 10 20 20 20 F F F
12832024-02-06T22:16:09.985Z 1 020-039 1 1 1 1 1 1 F F F
12842024-02-06T22:16:09.985Z 2 040-059 11 11 11 23 23 23 F F F
12852024-02-06T22:16:09.985Z 3 060-079 9 9 9 18 18 18 F F F
12862024-02-06T22:16:09.986Z 4 080-099 8 8 8 16 16 16 F F F
12872024-02-06T22:16:09.986Z 5 100-119 7 7 7 13 13 13 F F F
12882024-02-06T22:16:09.986Z 6 120-139 11 11 11 23 23 23 F F F
12892024-02-06T22:16:09.986Z 7 140-159 10 10 10 22 22 22 F F F
12902024-02-06T22:16:09.986Z 8 160-179 10 10 10 22 22 22 F F F
12912024-02-06T22:16:09.987Z 9 180-199 1 1 1 1 1 1 F F F
12922024-02-06T22:16:09.987Z 10 200-219 10 10 10 20 20 20 F F F
12932024-02-06T22:16:09.987Z 11 220-239 8 8 8 17 17 17 F F F
12942024-02-06T22:16:09.987Z 12 240-259 11 11 11 23 23 23 F F F
12952024-02-06T22:16:09.987Z 13 260-279 10 10 10 19 19 19 F F F
12962024-02-06T22:16:09.987Z 14 280-299 11 11 11 23 23 23 F F F
12972024-02-06T22:16:09.987Z 15 300-319 11 11 11 23 23 23 F F F
12982024-02-06T22:16:09.988Z 16 320-339 11 11 11 23 23 23 F F F
12992024-02-06T22:16:09.988Z 17 340-359 11 11 11 23 23 23 F F F
13002024-02-06T22:16:09.988Z 18 360-379 11 11 11 23 23 23 F F F
13012024-02-06T22:16:09.988Z 19 380-399 6 6 6 12 12 12 F F F
13022024-02-06T22:16:09.988Z 20 400-419 11 11 11 23 23 23 F F F
13032024-02-06T22:16:09.988Z 21 420-439 4 4 4 8 8 8 F F F
13042024-02-06T22:16:09.989Z 22 440-459 4 4 4 8 8 8 F F F
13052024-02-06T22:16:09.989Z 23 460-479 5 5 5 11 11 11 F F F
13062024-02-06T22:16:09.989Z 24 480-499 8 8 8 14 14 14 F F F
13072024-02-06T22:16:09.989Z 25 500-519 11 11 11 23 23 23 F F F
13082024-02-06T22:16:09.989Z 26 520-539 4 4 4 10 10 10 F F F
13092024-02-06T22:16:09.990Z 27 540-559 9 9 9 18 18 18 F F F
13102024-02-06T22:16:09.990Z 28 560-579 10 10 10 22 22 22 F F F
13112024-02-06T22:16:09.990Z 29 580-599 8 8 8 17 17 17 F F F
13122024-02-06T22:16:09.990ZMax gen: 11, Max flush: 23
13132024-02-06T22:16:09.991Z
13142024-02-06T22:16:09.991ZBegin loop 6
13152024-02-06T22:16:09.991ZDownstairs to restart: 1
13162024-02-06T22:16:09.991Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124184186Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183}
13172024-02-06T22:16:09.991Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124458899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183}
13182024-02-06T22:16:09.992Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124488615Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183}
13192024-02-06T22:16:09.992Z{"msg":"Crucible 85786b32-a021-4e85-909f-3c4fa8aa37ad has session id: f60357e8-b14c-4033-8083-2f2640a1fe07","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124514721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"Crucible runtime is spawned
13202024-02-06T22:16:09.992Zf60357e8-b14c-4033-8083-2f2640a1fe07"}
13212024-02-06T22:16:09.992Z{"msg":"Upstairs opts: Upstairs UUID: 85786b32-a021-4e85-909f-3c4fa8aa37ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124565274Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13222024-02-06T22:16:09.992Z{"msg":"Crucible stats registered with UUID: 85786b32-a021-4e85-909f-3c4fa8aa37ad","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124592091Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13232024-02-06T22:16:09.993Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124615827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183}
13242024-02-06T22:16:09.993Z{"msg":"85786b32-a021-4e85-909f-3c4fa8aa37ad active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124683958Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13252024-02-06T22:16:09.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.12474702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13262024-02-06T22:16:09.993Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124803142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13272024-02-06T22:16:09.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124862764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13282024-02-06T22:16:09.993Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124912597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13292024-02-06T22:16:09.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.124953162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13302024-02-06T22:16:09.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125054608Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13312024-02-06T22:16:09.994Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125104451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13322024-02-06T22:16:09.994Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125179231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13332024-02-06T22:16:09.994Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125209647Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13342024-02-06T22:16:09.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125236274Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13352024-02-06T22:16:09.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125253191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13362024-02-06T22:16:09.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125277128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13372024-02-06T22:16:09.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125776661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13382024-02-06T22:16:09.994Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125828124Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13392024-02-06T22:16:09.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125857899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13402024-02-06T22:16:09.994Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.125886226Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13412024-02-06T22:16:09.994Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.126340884Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13422024-02-06T22:16:09.995Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.126533078Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13432024-02-06T22:16:09.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.126858095Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13442024-02-06T22:16:09.995Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.126898879Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13452024-02-06T22:16:09.995Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.126957371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13462024-02-06T22:16:09.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.126994116Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13472024-02-06T22:16:09.995Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127026602Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13482024-02-06T22:16:09.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127102981Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13492024-02-06T22:16:09.995Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127131758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13502024-02-06T22:16:09.995Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127147235Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13512024-02-06T22:16:09.995Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127467142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13522024-02-06T22:16:09.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127903433Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13532024-02-06T22:16:09.995Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.12793037Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13542024-02-06T22:16:09.995Z{"msg":"[0]R flush_numbers[0..12]: [20, 1, 23, 18, 16, 13, 23, 22, 22, 1, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127945128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13552024-02-06T22:16:09.996Z{"msg":"[0]R generation[0..12]: [10, 1, 11, 9, 8, 7, 11, 10, 10, 1, 10, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127959236Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13562024-02-06T22:16:09.996Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127976933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13572024-02-06T22:16:09.996Z{"msg":"[1]R flush_numbers[0..12]: [20, 1, 23, 18, 16, 13, 23, 22, 22, 1, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.127991611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13582024-02-06T22:16:09.996Z{"msg":"[1]R generation[0..12]: [10, 1, 11, 9, 8, 7, 11, 10, 10, 1, 10, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128013148Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13592024-02-06T22:16:09.996Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128027956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13602024-02-06T22:16:09.996Z{"msg":"[2]R flush_numbers[0..12]: [20, 1, 23, 18, 16, 13, 23, 22, 22, 1, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128042184Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13612024-02-06T22:16:09.996Z{"msg":"[2]R generation[0..12]: [10, 1, 11, 9, 8, 7, 11, 10, 10, 1, 10, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128056073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13622024-02-06T22:16:09.996Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128071791Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13632024-02-06T22:16:09.996Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128093008Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13642024-02-06T22:16:09.996Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128108255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13652024-02-06T22:16:09.996Z{"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128121534Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13662024-02-06T22:16:09.996ZWait for a query_work_queue command to finish before sending IO
13672024-02-06T22:16:09.997Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.12814639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13682024-02-06T22:16:09.997Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128160748Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13692024-02-06T22:16:09.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128174037Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13702024-02-06T22:16:09.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128195044Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13712024-02-06T22:16:09.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128208812Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13722024-02-06T22:16:09.997Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.12822216Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13732024-02-06T22:16:09.997Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128241537Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13742024-02-06T22:16:09.997Z{"msg":"85786b32-a021-4e85-909f-3c4fa8aa37ad is now active with session: f60357e8-b14c-4033-8083-2f2640a1fe07","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128254966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13752024-02-06T22:16:09.997Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128266984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183,"session_id":"f60357e8-b14c-4033-8083-2f2640a1fe07"}
13762024-02-06T22:16:09.997Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:09.128279203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183}
13772024-02-06T22:16:09.997ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13782024-02-06T22:16:09.997ZLoading write count information from file "/tmp/test_repair_verify.data"
13792024-02-06T22:16:09.997ZRun Repair workload
13802024-02-06T22:16:09.998Z01/30 Write block 590 len 36864 data: 4 4 4 4 3 3 3 3 3
13812024-02-06T22:16:10.006Z02/30 Write block 497 len 20480 data: 2 2 2 2 2
13822024-02-06T22:16:10.006Z03/30 Read block 530 len 12288
13832024-02-06T22:16:10.006Z04/30 Read block 405 len 8192
13842024-02-06T22:16:10.006Z05/30 Write block 74 len 4096 data: 3
13852024-02-06T22:16:10.006Z06/30 Read block 379 len 40960
13862024-02-06T22:16:10.006Z07/30 Write block 374 len 28672 data: 3 3 4 5 5 4 4
13872024-02-06T22:16:10.006Z08/30 Read block 581 len 4096
13882024-02-06T22:16:10.006Z09/30 Write block 26 len 12288 data: 2 2 2
13892024-02-06T22:16:10.007Z10/30 Read block 146 len 4096
13902024-02-06T22:16:10.007Z11/30 Write block 52 len 28672 data: 6 6 6 6 3 3 3
13912024-02-06T22:16:10.007Z12/30 Read block 480 len 4096
13922024-02-06T22:16:10.007Z13/30 Flush
13932024-02-06T22:16:10.007Z14/30 Write block 434 len 40960 data: 3 3 3 3 3 3 3 2 2 2
13942024-02-06T22:16:10.007Z15/30 Write block 486 len 4096 data: 3
13952024-02-06T22:16:10.007Z16/30 Read block 45 len 16384
13962024-02-06T22:16:10.007Z17/30 Write block 526 len 28672 data: 3 3 3 2 2 2 2
13972024-02-06T22:16:10.007Z18/30 Write block 118 len 8192 data: 4 4
13982024-02-06T22:16:10.007Z19/30 Write block 114 len 32768 data: 3 4 3 4 5 5 3 4
13992024-02-06T22:16:10.007Z20/30 Write block 590 len 8192 data: 5 5
14002024-02-06T22:16:10.007Z21/30 Read block 112 len 4096
14012024-02-06T22:16:10.007Z22/30 Read block 538 len 36864
14022024-02-06T22:16:10.007Z23/30 Write block 148 len 32768 data: 5 4 4 2 2 4 3 3
14032024-02-06T22:16:10.008Z24/30 Read block 538 len 40960
14042024-02-06T22:16:10.008Z25/30 Write block 551 len 16384 data: 2 2 2 2
14052024-02-06T22:16:10.008Z26/30 Write block 385 len 32768 data: 2 2 2 2 2 2 2 2
14062024-02-06T22:16:10.008Z27/30 Write block 442 len 4096 data: 3
14072024-02-06T22:16:10.008Z28/30 Read block 493 len 32768
14082024-02-06T22:16:10.008Z29/30 Write block 494 len 40960 data: 2 2 2 3 3 3 3 3 3 3
14092024-02-06T22:16:10.008Z30/30 Write block 411 len 20480 data: 2 2 2 2 2
14102024-02-06T22:16:10.008Z----------------------------------------------------------------
14112024-02-06T22:16:10.008Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14122024-02-06T22:16:10.008ZStates: Active Active Active
14132024-02-06T22:16:10.008ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14142024-02-06T22:16:10.008Z 1 Acked 1000 Write 9 Done Done Done false
14152024-02-06T22:16:10.008Z 2 Acked 1001 Write 5 Done Done Done false
14162024-02-06T22:16:10.009Z 3 Acked 1002 Read 3 Done Done Done false
14172024-02-06T22:16:10.009Z 4 Acked 1003 Read 2 Done Done Done false
14182024-02-06T22:16:10.009Z 5 Acked 1004 Write 1 Done Done Done false
14192024-02-06T22:16:10.009Z 6 Acked 1005 Read 10 Done Done Done false
14202024-02-06T22:16:10.009Z 7 Acked 1006 Write 7 Done Done Done false
14212024-02-06T22:16:10.009Z 8 Acked 1007 Read 1 Done Done Done false
14222024-02-06T22:16:10.009Z 9 Acked 1008 Write 3 Done Done Done false
14232024-02-06T22:16:10.009Z 10 Acked 1009 Read 1 Done Done Done false
14242024-02-06T22:16:10.009Z 11 Acked 1010 Write 7 Done Sent Done false
14252024-02-06T22:16:10.009Z 12 Acked 1011 Read 1 Done Sent Done false
14262024-02-06T22:16:10.009Z 13 Acked 1012 Flush 0 Done Sent Done false
14272024-02-06T22:16:10.009Z 14 Acked 1013 Write 10 Done Sent Done false
14282024-02-06T22:16:10.009Z 15 Acked 1014 Write 1 Done Sent Done false
14292024-02-06T22:16:10.010Z 16 Acked 1015 Read 4 Done Sent Done false
14302024-02-06T22:16:10.010Z 17 Acked 1016 Write 7 Done Sent Done false
14312024-02-06T22:16:10.010Z 18 Acked 1017 Write 2 Done Sent Done false
14322024-02-06T22:16:10.010Z 19 Acked 1018 Write 8 Done Sent Done false
14332024-02-06T22:16:10.010Z 20 Acked 1019 Write 2 Done Sent Done false
14342024-02-06T22:16:10.010Z 21 Acked 1020 Read 1 Done Sent Done false
14352024-02-06T22:16:10.010Z 22 Acked 1021 Read 9 Done Sent Done false
14362024-02-06T22:16:10.010Z 23 Acked 1022 Write 8 Done Sent Done false
14372024-02-06T22:16:10.010Z 24 Acked 1023 Read 10 Done Sent Done false
14382024-02-06T22:16:10.011Z 25 Acked 1024 Write 4 Done Sent Done false
14392024-02-06T22:16:10.011Z 26 Acked 1025 Write 8 Done Sent Done false
14402024-02-06T22:16:10.011Z 27 Acked 1026 Write 1 Done Sent Done false
14412024-02-06T22:16:10.011Z 28 Acked 1027 Read 8 Done Sent Done false
14422024-02-06T22:16:10.011Z 29 Acked 1028 Write 10 Sent Sent Sent false
14432024-02-06T22:16:10.011Z 30 Acked 1029 Write 5 Sent Sent Sent false
14442024-02-06T22:16:10.011Z STATES DS:0 DS:1 DS:2 TOTAL
14452024-02-06T22:16:10.011Z New 0 0 0 0
14462024-02-06T22:16:10.011Z Sent 2 20 2 24
14472024-02-06T22:16:10.012Z Done 28 10 28 66
14482024-02-06T22:16:10.012Z Skipped 0 0 0 0
14492024-02-06T22:16:10.012Z Error 0 0 0 0
14502024-02-06T22:16:10.012ZLast Flush: 1012 0 1012
14512024-02-06T22:16:10.012ZDownstairs last five completed:
14522024-02-06T22:16:10.012ZUpstairs last five completed: 30 29 28 27 26
14532024-02-06T22:16:10.012Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:09.156261292Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1183}
14542024-02-06T22:16:10.012ZWrote out file "/tmp/test_repair_verify.data"
14552024-02-06T22:16:10.013Z
14562024-02-06T22:16:10.013ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14572024-02-06T22:16:10.013ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14582024-02-06T22:16:10.013Z 0 000-019 10 10 10 20 20 20 F F F
14592024-02-06T22:16:10.013Z 1 020-039 12 1 12 24 1 24 F T F <---
14602024-02-06T22:16:10.013Z 2 040-059 12 11 12 24 23 24 F F F <---
14612024-02-06T22:16:10.013Z 3 060-079 12 9 12 24 18 24 F T F <---
14622024-02-06T22:16:10.013Z 4 080-099 8 8 8 16 16 16 F F F
14632024-02-06T22:16:10.013Z 5 100-119 7 7 7 13 13 13 T F T <---
14642024-02-06T22:16:10.014Z 6 120-139 11 11 11 23 23 23 T F T <---
14652024-02-06T22:16:10.014Z 7 140-159 10 10 10 22 22 22 T F T <---
14662024-02-06T22:16:10.014Z 8 160-179 10 10 10 22 22 22 F F F
14672024-02-06T22:16:10.014Z 9 180-199 1 1 1 1 1 1 F F F
14682024-02-06T22:16:10.014Z 10 200-219 10 10 10 20 20 20 F F F
14692024-02-06T22:16:10.014Z 11 220-239 8 8 8 17 17 17 F F F
14702024-02-06T22:16:10.014Z 12 240-259 11 11 11 23 23 23 F F F
14712024-02-06T22:16:10.014Z 13 260-279 10 10 10 19 19 19 F F F
14722024-02-06T22:16:10.014Z 14 280-299 11 11 11 23 23 23 F F F
14732024-02-06T22:16:10.015Z 15 300-319 11 11 11 23 23 23 F F F
14742024-02-06T22:16:10.015Z 16 320-339 11 11 11 23 23 23 F F F
14752024-02-06T22:16:10.015Z 17 340-359 11 11 11 23 23 23 F F F
14762024-02-06T22:16:10.015Z 18 360-379 12 11 12 24 23 24 F T F <---
14772024-02-06T22:16:10.015Z 19 380-399 12 6 12 24 12 24 T T T <---
14782024-02-06T22:16:10.015Z 20 400-419 11 11 11 23 23 23 T F T <---
14792024-02-06T22:16:10.015Z 21 420-439 4 4 4 8 8 8 T F T <---
14802024-02-06T22:16:10.015Z 22 440-459 4 4 4 8 8 8 T F T <---
14812024-02-06T22:16:10.021Z 23 460-479 5 5 5 11 11 11 F F F
14822024-02-06T22:16:10.021Z 24 480-499 12 8 12 24 14 24 T T T <---
14832024-02-06T22:16:10.021Z 25 500-519 12 11 12 24 23 24 T T T <---
14842024-02-06T22:16:10.021Z 26 520-539 4 4 4 10 10 10 T F T <---
14852024-02-06T22:16:10.021Z 27 540-559 9 9 9 18 18 18 T F T <---
14862024-02-06T22:16:10.021Z 28 560-579 10 10 10 22 22 22 F F F
14872024-02-06T22:16:10.021Z 29 580-599 12 8 12 24 17 24 T T T <---
14882024-02-06T22:16:10.021ZMax gen: 12, Max flush: 24
14892024-02-06T22:16:10.022ZError: Difference in extent metadata found!
14902024-02-06T22:16:10.022ZA Difference in extent metadata is expected here
14912024-02-06T22:16:10.022ZOn loop 6
14922024-02-06T22:16:10.022Z
14932024-02-06T22:16:10.022ZVerifying data now
14942024-02-06T22:16:15.727ZNo change after verify
14952024-02-06T22:16:15.727ZLoop: 6 Downstairs dump after verify (and repair):
14962024-02-06T22:16:15.746ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14972024-02-06T22:16:15.746Z 0 000-019 10 10 10 20 20 20 F F F
14982024-02-06T22:16:15.746Z 1 020-039 12 12 12 24 24 24 F F F
14992024-02-06T22:16:15.746Z 2 040-059 12 12 12 24 24 24 F F F
15002024-02-06T22:16:15.746Z 3 060-079 12 12 12 24 24 24 F F F
15012024-02-06T22:16:15.746Z 4 080-099 8 8 8 16 16 16 F F F
15022024-02-06T22:16:15.746Z 5 100-119 13 13 13 25 25 25 F F F
15032024-02-06T22:16:15.746Z 6 120-139 13 13 13 25 25 25 F F F
15042024-02-06T22:16:15.746Z 7 140-159 13 13 13 25 25 25 F F F
15052024-02-06T22:16:15.746Z 8 160-179 10 10 10 22 22 22 F F F
15062024-02-06T22:16:15.746Z 9 180-199 1 1 1 1 1 1 F F F
15072024-02-06T22:16:15.746Z 10 200-219 10 10 10 20 20 20 F F F
15082024-02-06T22:16:15.746Z 11 220-239 8 8 8 17 17 17 F F F
15092024-02-06T22:16:15.746Z 12 240-259 11 11 11 23 23 23 F F F
15102024-02-06T22:16:15.757Z 13 260-279 10 10 10 19 19 19 F F F
15112024-02-06T22:16:15.757Z 14 280-299 11 11 11 23 23 23 F F F
15122024-02-06T22:16:15.757Z 15 300-319 11 11 11 23 23 23 F F F
15132024-02-06T22:16:15.757Z 16 320-339 11 11 11 23 23 23 F F F
15142024-02-06T22:16:15.757Z 17 340-359 11 11 11 23 23 23 F F F
15152024-02-06T22:16:15.757Z 18 360-379 12 12 12 24 24 24 F F F
15162024-02-06T22:16:15.757Z 19 380-399 13 13 13 25 25 25 F F F
15172024-02-06T22:16:15.757Z 20 400-419 13 13 13 25 25 25 F F F
15182024-02-06T22:16:15.757Z 21 420-439 13 13 13 25 25 25 F F F
15192024-02-06T22:16:15.757Z 22 440-459 13 13 13 25 25 25 F F F
15202024-02-06T22:16:15.757Z 23 460-479 5 5 5 11 11 11 F F F
15212024-02-06T22:16:15.757Z 24 480-499 13 13 13 25 25 25 F F F
15222024-02-06T22:16:15.757Z 25 500-519 13 13 13 25 25 25 F F F
15232024-02-06T22:16:15.757Z 26 520-539 13 13 13 25 25 25 F F F
15242024-02-06T22:16:15.757Z 27 540-559 13 13 13 25 25 25 F F F
15252024-02-06T22:16:15.758Z 28 560-579 10 10 10 22 22 22 F F F
15262024-02-06T22:16:15.758Z 29 580-599 13 13 13 25 25 25 F F F
15272024-02-06T22:16:15.758ZMax gen: 13, Max flush: 25
15282024-02-06T22:16:15.758Z
15292024-02-06T22:16:15.758ZBegin loop 7
15302024-02-06T22:16:15.758ZDownstairs to restart: 0
15312024-02-06T22:16:15.776Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.00207705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191}
15322024-02-06T22:16:15.776ZCrucible runtime is spawned
15332024-02-06T22:16:15.776Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002400986Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191}
15342024-02-06T22:16:15.776Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002453999Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191}
15352024-02-06T22:16:15.776Z{"msg":"Crucible 85ab0a99-c74c-4d1b-8e84-47002eec70f2 has session id: 248f1252-3de9-4ce2-ace5-0b4599a891b5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002484725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15362024-02-06T22:16:15.776Z{"msg":"Upstairs opts: Upstairs UUID: 85ab0a99-c74c-4d1b-8e84-47002eec70f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002542897Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15372024-02-06T22:16:15.776Z{"msg":"Crucible stats registered with UUID: 85ab0a99-c74c-4d1b-8e84-47002eec70f2","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002589521Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15382024-02-06T22:16:15.776Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002631975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191}
15392024-02-06T22:16:15.776Z{"msg":"85ab0a99-c74c-4d1b-8e84-47002eec70f2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002659661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15402024-02-06T22:16:15.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002700426Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15412024-02-06T22:16:15.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002737071Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15422024-02-06T22:16:15.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002795653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15432024-02-06T22:16:15.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002831368Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15442024-02-06T22:16:15.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002865583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15452024-02-06T22:16:15.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002913587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15462024-02-06T22:16:15.776Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002960341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15472024-02-06T22:16:15.777Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.002994296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15482024-02-06T22:16:15.777Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003030641Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15492024-02-06T22:16:15.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003056228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15502024-02-06T22:16:15.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003094332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15512024-02-06T22:16:15.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003132737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15522024-02-06T22:16:15.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003624471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15532024-02-06T22:16:15.777Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003665305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15542024-02-06T22:16:15.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003712169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15552024-02-06T22:16:15.777Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.003744335Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15562024-02-06T22:16:15.777Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004228979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15572024-02-06T22:16:15.777Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004431752Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15582024-02-06T22:16:15.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004809831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15592024-02-06T22:16:15.809Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004867203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15602024-02-06T22:16:15.809Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004898769Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15612024-02-06T22:16:15.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004934814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15622024-02-06T22:16:15.809Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004956531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15632024-02-06T22:16:15.809Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.004986777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15642024-02-06T22:16:15.809Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.005068606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15652024-02-06T22:16:15.809Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.005097492Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15662024-02-06T22:16:15.809Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.005567378Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15672024-02-06T22:16:15.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.005971214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15682024-02-06T22:16:15.809Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006004989Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15692024-02-06T22:16:15.809Z{"msg":"[0]R flush_numbers[0..12]: [20, 24, 24, 24, 16, 25, 25, 25, 22, 1, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006034775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15702024-02-06T22:16:15.809Z{"msg":"[0]R generation[0..12]: [10, 12, 12, 12, 8, 13, 13, 13, 10, 1, 10, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006060102Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15712024-02-06T22:16:15.810Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006100706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15722024-02-06T22:16:15.810Z{"msg":"[1]R flush_numbers[0..12]: [20, 24, 24, 24, 16, 25, 25, 25, 22, 1, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006117604Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15732024-02-06T22:16:15.810Z{"msg":"[1]R generation[0..12]: [10, 12, 12, 12, 8, 13, 13, 13, 10, 1, 10, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006140221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15742024-02-06T22:16:15.810Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006155379Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15752024-02-06T22:16:15.810Z{"msg":"[2]R flush_numbers[0..12]: [20, 24, 24, 24, 16, 25, 25, 25, 22, 1, 20, 17]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15762024-02-06T22:16:15.810Z,"time":"2024-02-06T22:16:15.006177576Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15772024-02-06T22:16:15.810Z{"msg":"[2]R generation[0..12]: [10, 12, 12, 12, 8, 13, 13, 13, 10, 1, 10, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006192454Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15782024-02-06T22:16:15.810Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006206952Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15792024-02-06T22:16:15.810Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.00622105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15802024-02-06T22:16:15.810Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006234928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15812024-02-06T22:16:15.810Z{"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006264934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15822024-02-06T22:16:15.810Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006281512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15832024-02-06T22:16:15.810Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006301729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15842024-02-06T22:16:15.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006326686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15852024-02-06T22:16:15.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006342424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15862024-02-06T22:16:15.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.00636897Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15872024-02-06T22:16:15.810Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006386798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15882024-02-06T22:16:15.810Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006409884Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15892024-02-06T22:16:15.810Z{"msg":"85ab0a99-c74c-4d1b-8e84-47002eec70f2 is now active with session: 248f1252-3de9-4ce2-ace5-0b4599a891b5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006424142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15902024-02-06T22:16:15.810Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006437771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191,"session_id":"248f1252-3de9-4ce2-ace5-0b4599a891b5"}
15912024-02-06T22:16:15.810Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:15.006462277Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191}
15922024-02-06T22:16:15.810ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15932024-02-06T22:16:15.810ZLoading write count information from file "/tmp/test_repair_verify.data"
15942024-02-06T22:16:15.810ZRun Repair workload
15952024-02-06T22:16:15.811Z01/30 Write block 220 len 36864 data: 3 3 2 2 2 3 3 3 3
15962024-02-06T22:16:15.811Z02/30 Read block 193 len 4096
15972024-02-06T22:16:15.811Z03/30 Write block 253 len 8192 data: 3 3
15982024-02-06T22:16:15.811Z04/30 Read block 92 len 28672
15992024-02-06T22:16:15.811Z05/30 Read block 594 len 20480
16002024-02-06T22:16:15.811Z06/30 Write block 122 len 20480 data: 4 3 3 4 4
16012024-02-06T22:16:15.811Z07/30 Read block 483 len 24576
16022024-02-06T22:16:15.811Z08/30 Read block 190 len 16384
16032024-02-06T22:16:15.811Z09/30 Flush
16042024-02-06T22:16:15.811Z10/30 Write block 416 len 24576 data: 2 3 3 2 2 2
16052024-02-06T22:16:15.811Z11/30 Read block 471 len 32768
16062024-02-06T22:16:15.811Z12/30 Flush
16072024-02-06T22:16:15.811Z13/30 Write block 49 len 12288 data: 4 4 6
16082024-02-06T22:16:15.811Z14/30 Write block 339 len 40960 data: 3 3 3 3 2 2 2 2 2 2
16092024-02-06T22:16:15.811Z15/30 Read block 430 len 28672
16102024-02-06T22:16:15.811Z16/30 Write block 179 len 28672 data: 2 2 2 2 2 2 2
16112024-02-06T22:16:15.811Z17/30 Write block 148 len 24576 data: 6 5 5 3 3 5
16122024-02-06T22:16:15.811Z18/30 Write block 423 len 36864 data: 2 2 2 2 2 2 2 2 2
16132024-02-06T22:16:15.811Z19/30 Read block 455 len 12288
16142024-02-06T22:16:15.811Z20/30 Flush
16152024-02-06T22:16:15.831Z21/30 Write block 202 len 32768 data: 3 3 3 3 3 3 3 3
16162024-02-06T22:16:15.831Z22/30 Write block 567 len 28672 data: 2 2 2 2 2 2 2
16172024-02-06T22:16:15.831Z23/30 Write block 366 len 24576 data: 3 3 2 2 2 2
16182024-02-06T22:16:15.831Z24/30 Read block 532 len 4096
16192024-02-06T22:16:15.831Z25/30 Read block 461 len 12288
16202024-02-06T22:16:15.831Z26/30 Read block 466 len 20480
16212024-02-06T22:16:15.831Z27/30 Read block 221 len 32768
16222024-02-06T22:16:15.831Z28/30 Read block 421 len 12288
16232024-02-06T22:16:15.832Z29/30 Write block 234 len 28672 data: 2 2 2 2 2 2 2
16242024-02-06T22:16:15.832Z30/30 Write block 55 len 12288 data: 7 4 4
16252024-02-06T22:16:15.832Z----------------------------------------------------------------
16262024-02-06T22:16:15.832Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16272024-02-06T22:16:15.832ZStates: Active Active Active
16282024-02-06T22:16:15.832ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16292024-02-06T22:16:15.832Z 1 Acked 1000 Write 9 Sent Done Done false
16302024-02-06T22:16:15.832Z 2 Acked 1001 Read 1 Sent Done Done false
16312024-02-06T22:16:15.832Z 3 Acked 1002 Write 2 Sent Done Done false
16322024-02-06T22:16:15.832Z 4 Acked 1003 Read 7 Sent Done Done false
16332024-02-06T22:16:15.832Z 5 Acked 1004 Read 5 Sent Done Done false
16342024-02-06T22:16:15.832Z 6 Acked 1005 Write 5 Sent Done Done false
16352024-02-06T22:16:15.832Z 7 Acked 1006 Read 6 Sent Done Done false
16362024-02-06T22:16:15.832Z 8 Acked 1007 Read 4 Sent Done Done false
16372024-02-06T22:16:15.832Z 9 Acked 1008 Flush 0 Sent Done Done false
16382024-02-06T22:16:15.832Z 10 Acked 1009 Write 6 Sent Done Done false
16392024-02-06T22:16:15.833Z 11 Acked 1010 Read 8 Sent Done Done false
16402024-02-06T22:16:15.833Z 12 Acked 1011 Flush 0 Sent Done Done false
16412024-02-06T22:16:15.833Z 13 Acked 1012 Write 3 Sent Done Done false
16422024-02-06T22:16:15.833Z 14 Acked 1013 Write 10 Sent Done Done false
16432024-02-06T22:16:15.833Z 15 Acked 1014 Read 7 Sent Done Done false
16442024-02-06T22:16:15.833Z 16 Acked 1015 Write 7 Sent Done Done false
16452024-02-06T22:16:15.833Z 17 Acked 1016 Write 6 Sent Done Done false
16462024-02-06T22:16:15.833Z 18 Acked 1017 Write 9 Sent Done Done false
16472024-02-06T22:16:15.833Z 19 Acked 1018 Read 3 Sent Done Done false
16482024-02-06T22:16:15.833Z 20 Acked 1019 Flush 0 Sent Done Done false
16492024-02-06T22:16:15.833Z 21 Acked 1020 Write 8 Sent Done Done false
16502024-02-06T22:16:15.833Z 22 Acked 1021 Write 7 Sent Done Done false
16512024-02-06T22:16:15.833Z 23 Acked 1022 Write 6 Sent Done Done false
16522024-02-06T22:16:15.833Z 24 Acked 1023 Read 1 Sent Done Done false
16532024-02-06T22:16:15.833Z 25 Acked 1024 Read 3 Sent Done Done false
16542024-02-06T22:16:15.833Z 26 Acked 1025 Read 5 Sent Done Done false
16552024-02-06T22:16:15.834Z 27 Acked 1026 Read 8 Sent Done Done false
16562024-02-06T22:16:15.834Z 28 Acked 1027 Read 3 Sent Done Done false
16572024-02-06T22:16:15.834Z 29 Acked 1028 Write 7 Sent Sent Sent false
16582024-02-06T22:16:15.834Z 30 Acked 1029 Write 3 Sent Sent Sent false
16592024-02-06T22:16:15.834Z STATES DS:0 DS:1 DS:2 TOTAL
16602024-02-06T22:16:15.834Z New 0 0 0 0
16612024-02-06T22:16:15.834Z Sent 30 2 2 34
16622024-02-06T22:16:15.834Z Done 0 28 28 56
16632024-02-06T22:16:15.834Z Skipped 0 0 0 0
16642024-02-06T22:16:15.834Z Error 0 0 0 0
16652024-02-06T22:16:15.834ZLast Flush: 0 1019 1019
16662024-02-06T22:16:15.834ZDownstairs last five completed:
16672024-02-06T22:16:15.834ZUpstairs last five completed: 30 29 28 27 26
16682024-02-06T22:16:15.834Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:15.039042126Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1191}
16692024-02-06T22:16:15.834ZWrote out file "/tmp/test_repair_verify.data"
16702024-02-06T22:16:15.835Z
16712024-02-06T22:16:15.835ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16722024-02-06T22:16:15.851ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16732024-02-06T22:16:15.851Z 0 000-019 10 10 10 20 20 20 F F F
16742024-02-06T22:16:15.851Z 1 020-039 12 12 12 24 24 24 F F F
16752024-02-06T22:16:15.851Z 2 040-059 12 14 14 24 28 28 F T T <---
16762024-02-06T22:16:15.851Z 3 060-079 12 12 12 24 24 24 F F F
16772024-02-06T22:16:15.851Z 4 080-099 8 8 8 16 16 16 F F F
16782024-02-06T22:16:15.851Z 5 100-119 13 13 13 25 25 25 F F F
16792024-02-06T22:16:15.851Z 6 120-139 13 14 14 25 26 26 F F F <---
16802024-02-06T22:16:15.851Z 7 140-159 13 14 14 25 28 28 F F F <---
16812024-02-06T22:16:15.851Z 8 160-179 10 14 14 22 28 28 F F F <---
16822024-02-06T22:16:15.852Z 9 180-199 1 14 14 1 28 28 F F F <---
16832024-02-06T22:16:15.852Z 10 200-219 10 10 10 20 20 20 F T T <---
16842024-02-06T22:16:15.852Z 11 220-239 8 14 14 17 26 26 F T T <---
16852024-02-06T22:16:15.852Z 12 240-259 11 14 14 23 26 26 F T T <---
16862024-02-06T22:16:15.852Z 13 260-279 10 10 10 19 19 19 F F F
16872024-02-06T22:16:15.852Z 14 280-299 11 11 11 23 23 23 F F F
16882024-02-06T22:16:15.852Z 15 300-319 11 11 11 23 23 23 F F F
16892024-02-06T22:16:15.852Z 16 320-339 11 14 14 23 28 28 F F F <---
16902024-02-06T22:16:15.852Z 17 340-359 11 14 14 23 28 28 F F F <---
16912024-02-06T22:16:15.852Z 18 360-379 12 12 12 24 24 24 F T T <---
16922024-02-06T22:16:15.852Z 19 380-399 13 13 13 25 25 25 F F F
16932024-02-06T22:16:15.852Z 20 400-419 13 14 14 25 27 27 F F F <---
16942024-02-06T22:16:15.852Z 21 420-439 13 14 14 25 28 28 F F F <---
16952024-02-06T22:16:15.852Z 22 440-459 13 13 13 25 25 25 F F F
16962024-02-06T22:16:15.852Z 23 460-479 5 5 5 11 11 11 F F F
16972024-02-06T22:16:15.852Z 24 480-499 13 13 13 25 25 25 F F F
16982024-02-06T22:16:15.853Z 25 500-519 13 13 13 25 25 25 F F F
16992024-02-06T22:16:15.853Z 26 520-539 13 13 13 25 25 25 F F F
17002024-02-06T22:16:15.853Z 27 540-559 13 13 13 25 25 25 F F F
17012024-02-06T22:16:15.853Z 28 560-579 10 10 10 22 22 22 F T T <---
17022024-02-06T22:16:15.853Z 29 580-599 13 13 13 25 25 25 F F F
17032024-02-06T22:16:15.853ZMax gen: 14, Max flush: 28
17042024-02-06T22:16:15.853ZError: Difference in extent metadata found!
17052024-02-06T22:16:15.853ZA Difference in extent metadata is expected here
17062024-02-06T22:16:15.853ZOn loop 7
17072024-02-06T22:16:15.853Z
17082024-02-06T22:16:15.853ZVerifying data now
17092024-02-06T22:16:20.518ZNo change after verify
17102024-02-06T22:16:20.518ZLoop: 7 Downstairs dump after verify (and repair):
17112024-02-06T22:16:20.968ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17122024-02-06T22:16:20.968Z 0 000-019 10 10 10 20 20 20 F F F
17132024-02-06T22:16:20.968Z 1 020-039 12 12 12 24 24 24 F F F
17142024-02-06T22:16:20.968Z 2 040-059 15 15 15 29 29 29 F F F
17152024-02-06T22:16:20.968Z 3 060-079 12 12 12 24 24 24 F F F
17162024-02-06T22:16:20.969Z 4 080-099 8 8 8 16 16 16 F F F
17172024-02-06T22:16:20.969Z 5 100-119 13 13 13 25 25 25 F F F
17182024-02-06T22:16:20.969Z 6 120-139 14 14 14 26 26 26 F F F
17192024-02-06T22:16:20.969Z 7 140-159 14 14 14 28 28 28 F F F
17202024-02-06T22:16:20.969Z 8 160-179 14 14 14 28 28 28 F F F
17212024-02-06T22:16:20.969Z 9 180-199 14 14 14 28 28 28 F F F
17222024-02-06T22:16:20.969Z 10 200-219 15 15 15 29 29 29 F F F
17232024-02-06T22:16:20.969Z 11 220-239 15 15 15 29 29 29 F F F
17242024-02-06T22:16:20.969Z 12 240-259 15 15 15 29 29 29 F F F
17252024-02-06T22:16:20.969Z 13 260-279 10 10 10 19 19 19 F F F
17262024-02-06T22:16:20.969Z 14 280-299 11 11 11 23 23 23 F F F
17272024-02-06T22:16:20.969Z 15 300-319 11 11 11 23 23 23 F F F
17282024-02-06T22:16:20.969Z 16 320-339 14 14 14 28 28 28 F F F
17292024-02-06T22:16:20.969Z 17 340-359 14 14 14 28 28 28 F F F
17302024-02-06T22:16:20.969Z 18 360-379 15 15 15 29 29 29 F F F
17312024-02-06T22:16:20.969Z 19 380-399 13 13 13 25 25 25 F F F
17322024-02-06T22:16:20.969Z 20 400-419 14 14 14 27 27 27 F F F
17332024-02-06T22:16:20.969Z 21 420-439 14 14 14 28 28 28 F F F
17342024-02-06T22:16:20.969Z 22 440-459 13 13 13 25 25 25 F F F
17352024-02-06T22:16:20.969Z 23 460-479 5 5 5 11 11 11 F F F
17362024-02-06T22:16:20.969Z 24 480-499 13 13 13 25 25 25 F F F
17372024-02-06T22:16:20.969Z 25 500-519 13 13 13 25 25 25 F F F
17382024-02-06T22:16:20.969Z 26 520-539 13 13 13 25 25 25 F F F
17392024-02-06T22:16:20.969Z 27 540-559 13 13 13 25 25 25 F F F
17402024-02-06T22:16:20.969Z 28 560-579 15 15 15 29 29 29 F F F
17412024-02-06T22:16:20.970Z 29 580-599 13 13 13 25 25 25 F F F
17422024-02-06T22:16:20.970ZMax gen: 15, Max flush: 29
17432024-02-06T22:16:20.970Z
17442024-02-06T22:16:20.970ZBegin loop 8
17452024-02-06T22:16:20.970ZDownstairs to restart: 1
17462024-02-06T22:16:20.970Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789190422Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199}
17472024-02-06T22:16:20.970Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789455676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199}
17482024-02-06T22:16:20.970Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789491542Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199}
17492024-02-06T22:16:20.970Z{Crucible runtime is spawned
17502024-02-06T22:16:20.970Z"msg":"Crucible ba42e80a-b2ef-4c71-9a9d-9e7d8818accf has session id: 9fd32de5-8992-482a-90c3-24fdf8e23a87","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789520468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17512024-02-06T22:16:20.970Z{"msg":"Upstairs opts: Upstairs UUID: ba42e80a-b2ef-4c71-9a9d-9e7d8818accf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789566352Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17522024-02-06T22:16:20.970Z{"msg":"Crucible stats registered with UUID: ba42e80a-b2ef-4c71-9a9d-9e7d8818accf","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789600857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17532024-02-06T22:16:20.970Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789630423Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199}
17542024-02-06T22:16:20.970Z{"msg":"ba42e80a-b2ef-4c71-9a9d-9e7d8818accf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789647921Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17552024-02-06T22:16:20.970Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789685125Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17562024-02-06T22:16:20.970Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789712062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17572024-02-06T22:16:20.970Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789743848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17582024-02-06T22:16:20.970Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789769104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17592024-02-06T22:16:20.970Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789794451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17602024-02-06T22:16:20.970Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789819027Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17612024-02-06T22:16:20.970Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789842464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17622024-02-06T22:16:20.970Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.78987563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17632024-02-06T22:16:20.970Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.789959498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17642024-02-06T22:16:20.971Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.790012171Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17652024-02-06T22:16:20.971Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.790039277Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17662024-02-06T22:16:20.971Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.790063674Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17672024-02-06T22:16:20.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.790786927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17682024-02-06T22:16:20.971Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.790842139Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17692024-02-06T22:16:20.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.790875575Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17702024-02-06T22:16:20.971Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.790901231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17712024-02-06T22:16:20.971Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.791505119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17722024-02-06T22:16:20.971Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.791546864Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17732024-02-06T22:16:20.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.79157852Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17742024-02-06T22:16:20.971Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.791656149Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17752024-02-06T22:16:20.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.79216375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17762024-02-06T22:16:20.971Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792211324Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17772024-02-06T22:16:20.971Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792250699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17782024-02-06T22:16:20.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792278835Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17792024-02-06T22:16:20.971Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.79231514Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17802024-02-06T22:16:20.971Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792351505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17812024-02-06T22:16:20.971Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792376702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17822024-02-06T22:16:20.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792843819Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17832024-02-06T22:16:20.971Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792877794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17842024-02-06T22:16:20.971Z{"msg":"[0]R flush_numbers[0..12]: [20, 24, 29, 24, 16, 25, 26, 28, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792894292Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17852024-02-06T22:16:20.971Z{"msg":"[0]R generation[0..12]: [10, 12, 15, 12, 8, 13, 14, 14, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792916919Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17862024-02-06T22:16:20.971Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792932687Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17872024-02-06T22:16:20.971Z{"msg":"[1]R flush_numbers[0..12]: [20, 24, 29, 24, 16, 25, 26, 28, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792954944Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17882024-02-06T22:16:20.972Z{"msg":"[1]R generation[0..12]: [10, 12, 15, 12, 8, 13, 14, 14, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792969472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17892024-02-06T22:16:20.972Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.792990909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17902024-02-06T22:16:20.972Z{"msg":"[2]R flush_numbers[0..12]: [20, 24, 29, 24, 16, 25, 26, 28, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793006287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17912024-02-06T22:16:20.972Z{"msg":"[2]R generation[0..12]: [10, 12, 15, 12, 8, 13, 14, 14, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793020875Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17922024-02-06T22:16:20.972Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793035943Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17932024-02-06T22:16:20.972ZWait for a query_work_queue command to finish before sending IO
17942024-02-06T22:16:20.972Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.79305643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17952024-02-06T22:16:20.972Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793078527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17962024-02-06T22:16:20.972Z{"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793092935Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17972024-02-06T22:16:20.972Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793106843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17982024-02-06T22:16:20.972Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.79312763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
17992024-02-06T22:16:20.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793140738Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
18002024-02-06T22:16:20.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793161536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
18012024-02-06T22:16:20.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793175194Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
18022024-02-06T22:16:20.972Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793189732Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
18032024-02-06T22:16:20.972Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793209639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
18042024-02-06T22:16:20.972Z{"msg":"ba42e80a-b2ef-4c71-9a9d-9e7d8818accf is now active with session: 9fd32de5-8992-482a-90c3-24fdf8e23a87","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793223067Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
18052024-02-06T22:16:20.972Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793235786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199,"session_id":"9fd32de5-8992-482a-90c3-24fdf8e23a87"}
18062024-02-06T22:16:20.972Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:19.793255643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199}
18072024-02-06T22:16:20.972ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18082024-02-06T22:16:20.972ZLoading write count information from file "/tmp/test_repair_verify.data"
18092024-02-06T22:16:20.972ZRun Repair workload
18102024-02-06T22:16:20.972Z01/30 Write block 74 len 8192 data: 4 3
18112024-02-06T22:16:20.978Z02/30 Write block 249 len 36864 data: 5 4 3 3 4 4 3 2 2
18122024-02-06T22:16:20.978Z03/30 Read block 161 len 32768
18132024-02-06T22:16:20.978Z04/30 Write block 59 len 8192 data: 3 3
18142024-02-06T22:16:20.978Z05/30 Write block 124 len 32768 data: 4 5 5 4 4 3 3 3
18152024-02-06T22:16:20.978Z06/30 Read block 279 len 24576
18162024-02-06T22:16:20.978Z07/30 Flush
18172024-02-06T22:16:20.979Z08/30 Write block 558 len 8192 data: 3 3
18182024-02-06T22:16:20.979Z09/30 Read block 298 len 36864
18192024-02-06T22:16:20.979Z10/30 Flush
18202024-02-06T22:16:20.979Z11/30 Write block 326 len 36864 data: 3 3 2 2 2 2 2 2 2
18212024-02-06T22:16:20.979Z12/30 Read block 196 len 28672
18222024-02-06T22:16:20.979Z13/30 Write block 312 len 20480 data: 2 3 4 4 3
18232024-02-06T22:16:20.979Z14/30 Read block 133 len 36864
18242024-02-06T22:16:20.979Z15/30 Write block 98 len 16384 data: 2 2 2 2
18252024-02-06T22:16:20.979Z16/30 Flush
18262024-02-06T22:16:20.979Z17/30 Flush
18272024-02-06T22:16:20.979Z18/30 Write block 310 len 36864 data: 2 2 3 4 5 5 4 3 3
18282024-02-06T22:16:20.979Z19/30 Flush
18292024-02-06T22:16:20.979Z20/30 Write block 579 len 16384 data: 2 3 2 2
18302024-02-06T22:16:20.979Z21/30 Flush
18312024-02-06T22:16:20.979Z22/30 Flush
18322024-02-06T22:16:20.979Z23/30 Write block 476 len 16384 data: 3 3 3 2
18332024-02-06T22:16:20.979Z24/30 Write block 513 len 20480 data: 3 3 3 3 3
18342024-02-06T22:16:20.979Z25/30 Read block 368 len 32768
18352024-02-06T22:16:20.979Z26/30 Read block 163 len 28672
18362024-02-06T22:16:20.979Z27/30 Read block 208 len 16384
18372024-02-06T22:16:20.979Z28/30 Read block 520 len 40960
18382024-02-06T22:16:20.979Z29/30 Read block 139 len 8192
18392024-02-06T22:16:20.979Z30/30 Write block 360 len 24576 data: 4 4 4 4 3 3
18402024-02-06T22:16:20.979Z----------------------------------------------------------------
18412024-02-06T22:16:20.979Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18422024-02-06T22:16:20.980ZStates: Active Active Active
18432024-02-06T22:16:20.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18442024-02-06T22:16:20.980Z 1 Acked 1000 Write 2 Done Done Done false
18452024-02-06T22:16:20.980Z 2 Acked 1001 Write 9 Done Done Done false
18462024-02-06T22:16:20.980Z 3 Acked 1002 Read 8 Done Done Done false
18472024-02-06T22:16:20.980Z 4 Acked 1003 Write 2 Done Sent Done false
18482024-02-06T22:16:20.980Z 5 Acked 1004 Write 8 Done Sent Done false
18492024-02-06T22:16:20.980Z 6 Acked 1005 Read 6 Done Sent Done false
18502024-02-06T22:16:20.980Z 7 Acked 1006 Flush 0 Done Sent Done false
18512024-02-06T22:16:20.980Z 8 Acked 1007 Write 2 Done Sent Done false
18522024-02-06T22:16:20.980Z 9 Acked 1008 Read 9 Done Sent Done false
18532024-02-06T22:16:20.980Z 10 Acked 1009 Flush 0 Done Sent Done false
18542024-02-06T22:16:20.980Z 11 Acked 1010 Write 9 Done Sent Done false
18552024-02-06T22:16:20.980Z 12 Acked 1011 Read 7 Done Sent Done false
18562024-02-06T22:16:20.980Z 13 Acked 1012 Write 5 Done Sent Done false
18572024-02-06T22:16:20.980Z 14 Acked 1013 Read 9 Done Sent Done false
18582024-02-06T22:16:20.980Z 15 Acked 1014 Write 4 Done Sent Done false
18592024-02-06T22:16:20.980Z 16 Acked 1015 Flush 0 Done Sent Done false
18602024-02-06T22:16:20.980Z 17 Acked 1016 Flush 0 Done Sent Done false
18612024-02-06T22:16:20.980Z 18 Acked 1017 Write 9 Done Sent Done false
18622024-02-06T22:16:20.980Z 19 Acked 1018 Flush 0 Done Sent Done false
18632024-02-06T22:16:20.980Z 20 Acked 1019 Write 4 Done Sent Done false
18642024-02-06T22:16:20.980Z 21 Acked 1020 Flush 0 Done Sent Done false
18652024-02-06T22:16:20.980Z 22 Acked 1021 Flush 0 Done Sent Done false
18662024-02-06T22:16:20.980Z 23 Acked 1022 Write 4 Done Sent Done false
18672024-02-06T22:16:20.981Z 24 Acked 1023 Write 5 Done Sent Done false
18682024-02-06T22:16:20.981Z 25 Acked 1024 Read 8 Done Sent Done false
18692024-02-06T22:16:20.981Z 26 Acked 1025 Read 7 Done Sent Done false
18702024-02-06T22:16:20.981Z 27 Acked 1026 Read 4 Done Sent Done false
18712024-02-06T22:16:20.981Z 28 Acked 1027 Read 10 Done Sent Done false
18722024-02-06T22:16:20.981Z 29 Acked 1028 Read 2 Done Sent Done false
18732024-02-06T22:16:20.981Z 30 Acked 1029 Write 6 Sent Sent Sent false
18742024-02-06T22:16:20.981Z STATES DS:0 DS:1 DS:2 TOTAL
18752024-02-06T22:16:20.981Z New 0 0 0 0
18762024-02-06T22:16:20.981Z Sent 1 27 1 29
18772024-02-06T22:16:20.981Z Done 29 3 29 61
18782024-02-06T22:16:20.981Z Skipped 0 0 0 0
18792024-02-06T22:16:20.981Z Error 0 0 0 0
18802024-02-06T22:16:20.981ZLast Flush: 1021 0 1021
18812024-02-06T22:16:20.981ZDownstairs last five completed:
18822024-02-06T22:16:20.981ZUpstairs last five completed: 30 29 28 27 26
18832024-02-06T22:16:20.981Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:19.82591922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1199}
18842024-02-06T22:16:20.981ZWrote out file "/tmp/test_repair_verify.data"
18852024-02-06T22:16:20.981Z
18862024-02-06T22:16:20.981ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18872024-02-06T22:16:20.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18882024-02-06T22:16:20.981Z 0 000-019 10 10 10 20 20 20 F F F
18892024-02-06T22:16:20.981Z 1 020-039 12 12 12 24 24 24 F F F
18902024-02-06T22:16:20.981Z 2 040-059 16 15 16 30 29 30 F F F <---
18912024-02-06T22:16:20.981Z 3 060-079 16 12 16 30 24 30 F T F <---
18922024-02-06T22:16:20.982Z 4 080-099 16 8 16 32 16 32 F F F <---
18932024-02-06T22:16:20.982Z 5 100-119 16 13 16 32 25 32 F F F <---
18942024-02-06T22:16:20.982Z 6 120-139 16 14 16 30 26 30 F F F <---
18952024-02-06T22:16:20.982Z 7 140-159 14 14 14 28 28 28 F F F
18962024-02-06T22:16:20.982Z 8 160-179 14 14 14 28 28 28 F F F
18972024-02-06T22:16:20.982Z 9 180-199 14 14 14 28 28 28 F F F
18982024-02-06T22:16:20.982Z 10 200-219 15 15 15 29 29 29 F F F
18992024-02-06T22:16:20.982Z 11 220-239 15 15 15 29 29 29 F F F
19002024-02-06T22:16:20.982Z 12 240-259 16 15 16 30 29 30 F T F <---
19012024-02-06T22:16:20.982Z 13 260-279 10 10 10 19 19 19 F F F
19022024-02-06T22:16:20.982Z 14 280-299 11 11 11 23 23 23 F F F
19032024-02-06T22:16:20.982Z 15 300-319 16 11 16 34 23 34 F F F <---
19042024-02-06T22:16:20.982Z 16 320-339 16 14 16 32 28 32 F F F <---
19052024-02-06T22:16:20.982Z 17 340-359 14 14 14 28 28 28 F F F
19062024-02-06T22:16:20.982Z 18 360-379 15 15 15 29 29 29 T F T <---
19072024-02-06T22:16:20.982Z 19 380-399 13 13 13 25 25 25 F F F
19082024-02-06T22:16:20.982Z 20 400-419 14 14 14 27 27 27 F F F
19092024-02-06T22:16:20.982Z 21 420-439 14 14 14 28 28 28 F F F
19102024-02-06T22:16:20.982Z 22 440-459 13 13 13 25 25 25 F F F
19112024-02-06T22:16:20.989Z 23 460-479 5 5 5 11 11 11 T F T <---
19122024-02-06T22:16:20.989Z 24 480-499 13 13 13 25 25 25 F F F
19132024-02-06T22:16:20.989Z 25 500-519 13 13 13 25 25 25 T F T <---
19142024-02-06T22:16:20.989Z 26 520-539 13 13 13 25 25 25 F F F
19152024-02-06T22:16:20.989Z 27 540-559 16 13 16 31 25 31 F F F <---
19162024-02-06T22:16:20.989Z 28 560-579 16 15 16 35 29 35 F F F <---
19172024-02-06T22:16:20.989Z 29 580-599 16 13 16 35 25 35 F F F <---
19182024-02-06T22:16:20.989ZMax gen: 16, Max flush: 35
19192024-02-06T22:16:20.989ZError: Difference in extent metadata found!
19202024-02-06T22:16:20.989ZA Difference in extent metadata is expected here
19212024-02-06T22:16:20.989ZOn loop 8
19222024-02-06T22:16:20.989Z
19232024-02-06T22:16:20.989ZVerifying data now
19242024-02-06T22:16:26.301ZNo change after verify
19252024-02-06T22:16:26.302ZLoop: 8 Downstairs dump after verify (and repair):
19262024-02-06T22:16:26.877ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19272024-02-06T22:16:26.877Z 0 000-019 10 10 10 20 20 20 F F F
19282024-02-06T22:16:26.877Z 1 020-039 12 12 12 24 24 24 F F F
19292024-02-06T22:16:26.878Z 2 040-059 16 16 16 30 30 30 F F F
19302024-02-06T22:16:26.878Z 3 060-079 16 16 16 30 30 30 F F F
19312024-02-06T22:16:26.878Z 4 080-099 16 16 16 32 32 32 F F F
19322024-02-06T22:16:26.878Z 5 100-119 16 16 16 32 32 32 F F F
19332024-02-06T22:16:26.878Z 6 120-139 16 16 16 30 30 30 F F F
19342024-02-06T22:16:26.878Z 7 140-159 14 14 14 28 28 28 F F F
19352024-02-06T22:16:26.878Z 8 160-179 14 14 14 28 28 28 F F F
19362024-02-06T22:16:26.878Z 9 180-199 14 14 14 28 28 28 F F F
19372024-02-06T22:16:26.878Z 10 200-219 15 15 15 29 29 29 F F F
19382024-02-06T22:16:26.878Z 11 220-239 15 15 15 29 29 29 F F F
19392024-02-06T22:16:26.878Z 12 240-259 16 16 16 30 30 30 F F F
19402024-02-06T22:16:26.900Z 13 260-279 10 10 10 19 19 19 F F F
19412024-02-06T22:16:26.900Z 14 280-299 11 11 11 23 23 23 F F F
19422024-02-06T22:16:26.900Z 15 300-319 16 16 16 34 34 34 F F F
19432024-02-06T22:16:26.900Z 16 320-339 16 16 16 32 32 32 F F F
19442024-02-06T22:16:26.900Z 17 340-359 14 14 14 28 28 28 F F F
19452024-02-06T22:16:26.901Z 18 360-379 17 17 17 36 36 36 F F F
19462024-02-06T22:16:26.901Z 19 380-399 13 13 13 25 25 25 F F F
19472024-02-06T22:16:26.901Z 20 400-419 14 14 14 27 27 27 F F F
19482024-02-06T22:16:26.901Z 21 420-439 14 14 14 28 28 28 F F F
19492024-02-06T22:16:26.901Z 22 440-459 13 13 13 25 25 25 F F F
19502024-02-06T22:16:26.901Z 23 460-479 17 17 17 36 36 36 F F F
19512024-02-06T22:16:26.901Z 24 480-499 13 13 13 25 25 25 F F F
19522024-02-06T22:16:26.902Z 25 500-519 17 17 17 36 36 36 F F F
19532024-02-06T22:16:26.902Z 26 520-539 13 13 13 25 25 25 F F F
19542024-02-06T22:16:26.902Z 27 540-559 16 16 16 31 31 31 F F F
19552024-02-06T22:16:26.902Z 28 560-579 16 16 16 35 35 35 F F F
19562024-02-06T22:16:26.902Z 29 580-599 16 16 16 35 35 35 F F F
19572024-02-06T22:16:26.902ZMax gen: 17, Max flush: 36
19582024-02-06T22:16:26.902Z
19592024-02-06T22:16:26.902ZBegin loop 9
19602024-02-06T22:16:26.902ZDownstairs to restart: 1
19612024-02-06T22:16:26.903Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.569813878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207}
19622024-02-06T22:16:26.903Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570071464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207}
19632024-02-06T22:16:26.903Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570108599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207}
19642024-02-06T22:16:26.903Z{"msg":"Crucible c1708420-ef4f-43d4-9354-81dab9e4bae4 has session id: 080a4e18-4787-4368-8514-be440d650e2f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570127196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19652024-02-06T22:16:26.903ZCrucible runtime is spawned
19662024-02-06T22:16:26.903Z{"msg":"Upstairs opts: Upstairs UUID: c1708420-ef4f-43d4-9354-81dab9e4bae4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570181569Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19672024-02-06T22:16:26.903Z{"msg":"Crucible stats registered with UUID: c1708420-ef4f-43d4-9354-81dab9e4bae4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570210215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19682024-02-06T22:16:26.903Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570228793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207}
19692024-02-06T22:16:26.903Z{"msg":"c1708420-ef4f-43d4-9354-81dab9e4bae4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.57032219Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19702024-02-06T22:16:26.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570347656Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19712024-02-06T22:16:26.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570363884Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19722024-02-06T22:16:26.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570403579Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19732024-02-06T22:16:26.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570429955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19742024-02-06T22:16:26.904Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570461271Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19752024-02-06T22:16:26.904Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570486907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19762024-02-06T22:16:26.904Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570517623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19772024-02-06T22:16:26.904Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570549899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19782024-02-06T22:16:26.905Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570577795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19792024-02-06T22:16:26.905Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570723995Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19802024-02-06T22:16:26.905Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.570773099Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19812024-02-06T22:16:26.905Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:25.570900022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19822024-02-06T22:16:26.905Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:25.570942986Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19832024-02-06T22:16:26.905Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:25.57098793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19842024-02-06T22:16:26.905Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.571008247Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19852024-02-06T22:16:26.906Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.571037573Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19862024-02-06T22:16:26.906Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.571085097Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19872024-02-06T22:16:26.906Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.57157484Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19882024-02-06T22:16:26.906Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.571613135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19892024-02-06T22:16:26.906Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.571629943Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19902024-02-06T22:16:26.906Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.57165224Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19912024-02-06T22:16:26.906Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.572232322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19922024-02-06T22:16:26.906Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.572271876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19932024-02-06T22:16:26.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.572846359Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19942024-02-06T22:16:26.907Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.572884683Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19952024-02-06T22:16:26.907Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.57290808Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19962024-02-06T22:16:26.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.572958464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19972024-02-06T22:16:26.907Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.572975011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19982024-02-06T22:16:26.907Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:25.572988929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
19992024-02-06T22:16:27.400Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.572545747Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20002024-02-06T22:16:27.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.572800682Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20012024-02-06T22:16:27.414Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.573789928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20022024-02-06T22:16:27.414Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.573837402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20032024-02-06T22:16:27.414Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.574342654Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20042024-02-06T22:16:27.414Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.574861724Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20052024-02-06T22:16:27.414Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.574908048Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20062024-02-06T22:16:27.415Z{"msg":"[0]R flush_numbers[0..12]: [20, 24, 30, 30, 32, 32, 30, 28, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.574929845Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20072024-02-06T22:16:27.415Z{"msg":"[0]R generation[0..12]: [10, 12, 16, 16, 16, 16, 16, 14, 14, 14, 15, 15]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
20082024-02-06T22:16:27.415Z,"time":"2024-02-06T22:16:26.57495905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20092024-02-06T22:16:27.415Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.574982857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20102024-02-06T22:16:27.415Z{"msg":"[1]R flush_numbers[0..12]: [20, 24, 30, 30, 32, 32, 30, 28, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575019692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20112024-02-06T22:16:27.415Z{"msg":"[1]R generation[0..12]: [10, 12, 16, 16, 16, 16, 16, 14, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575065406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20122024-02-06T22:16:27.415Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.57510949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20132024-02-06T22:16:27.415Z{"msg":"[2]R flush_numbers[0..12]: [20, 24, 30, 30, 32, 32, 30, 28, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575156094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20142024-02-06T22:16:27.415Z{"msg":"[2]R generation[0..12]: [10, 12, 16, 16, 16, 16, 16, 14, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575194209Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20152024-02-06T22:16:27.415Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575235863Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20162024-02-06T22:16:27.415Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20172024-02-06T22:16:27.416Z2024-02-06T22:16:26.575279557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20182024-02-06T22:16:27.416Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575324031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20192024-02-06T22:16:27.416Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575358227Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20202024-02-06T22:16:27.416Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575386523Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20212024-02-06T22:16:27.416Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.57540486Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20222024-02-06T22:16:27.416Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575429547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20232024-02-06T22:16:27.416Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575448824Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20242024-02-06T22:16:27.416Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575470341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20252024-02-06T22:16:27.416Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575484759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20262024-02-06T22:16:27.416Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575498298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20272024-02-06T22:16:27.417Z{"msg":"c1708420-ef4f-43d4-9354-81dab9e4bae4 is now active with session: 080a4e18-4787-4368-8514-be440d650e2f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575519205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20282024-02-06T22:16:27.417Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575532453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207,"session_id":"080a4e18-4787-4368-8514-be440d650e2f"}
20292024-02-06T22:16:27.417Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:26.575552101Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207}
20302024-02-06T22:16:27.417ZLoading write count information from file "/tmp/test_repair_verify.data"
20312024-02-06T22:16:27.417ZRun Repair workload
20322024-02-06T22:16:27.417Z01/30 Write block 395 len 32768 data: 2 2 2 3 3 3 3 3
20332024-02-06T22:16:27.417Z02/30 Read block 414 len 16384
20342024-02-06T22:16:27.417Z03/30 Write block 117 len 28672 data: 5 6 6 4 5 5 4
20352024-02-06T22:16:27.417Z04/30 Write block 363 len 36864 data: 5 4 4 4 4 3 3 3 3
20362024-02-06T22:16:27.417Z05/30 Read block 474 len 12288
20372024-02-06T22:16:27.417Z06/30 Write block 574 len 32768 data: 2 2 2 2 2 3 4 3
20382024-02-06T22:16:27.417Z07/30 Write block 471 len 36864 data: 3 3 3 3 3 4 4 4 3
20392024-02-06T22:16:27.418Z08/30 Read block 22 len 4096
20402024-02-06T22:16:27.418Z09/30 Flush
20412024-02-06T22:16:27.418Z10/30 Write block 82 len 12288 data: 3 3 3
20422024-02-06T22:16:27.418Z11/30 Read block 374 len 12288
20432024-02-06T22:16:27.418Z12/30 Write block 241 len 24576 data: 2 3 3 3 3 3
20442024-02-06T22:16:27.418Z13/30 Read block 304 len 4096
20452024-02-06T22:16:27.418Z14/30 Write block 560 len 8192 data: 3 3
20462024-02-06T22:16:27.418Z15/30 Read block 243 len 12288
20472024-02-06T22:16:27.418Z16/30 Write block 530 len 20480 data: 3 3 3 2 2
20482024-02-06T22:16:27.418Z17/30 Read block 530 len 12288
20492024-02-06T22:16:27.418Z18/30 Read block 70 len 36864
20502024-02-06T22:16:27.418Z19/30 Flush
20512024-02-06T22:16:27.418Z20/30 Write block 25 len 28672 data: 2 3 3 3 2 2 2
20522024-02-06T22:16:27.418Z21/30 Read block 158 len 16384
20532024-02-06T22:16:27.419Z22/30 Flush
20542024-02-06T22:16:27.419Z23/30 Write block 127 len 8192 data: 5 5
20552024-02-06T22:16:27.419Z24/30 Write block 147 len 16384 data: 5 7 6 6
20562024-02-06T22:16:27.419Z25/30 Flush
20572024-02-06T22:16:27.419Z26/30 Write block 342 len 28672 data: 4 3 3 3 3 3 3
20582024-02-06T22:16:27.419Z27/30 Read block 207 len 4096
20592024-02-06T22:16:27.419Z28/30 Write block 334 len 28672 data: 3 2 3 3 4 4 4
20602024-02-06T22:16:27.419Z29/30 Read block 303 len 28672
20612024-02-06T22:16:27.419Z30/30 Write block 448 len 24576 data: 2 2 2 2 2 2
20622024-02-06T22:16:27.419Z----------------------------------------------------------------
20632024-02-06T22:16:27.419Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20642024-02-06T22:16:27.420ZStates: Active Active Active
20652024-02-06T22:16:27.420ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20662024-02-06T22:16:27.420Z 1 Acked 1000 Write 8 Done Done Done false
20672024-02-06T22:16:27.420Z 2 Acked 1001 Read 4 Done Done Done false
20682024-02-06T22:16:27.420Z 3 Acked 1002 Write 7 Done Sent Done false
20692024-02-06T22:16:27.420Z 4 Acked 1003 Write 9 Done Sent Done false
20702024-02-06T22:16:27.420Z 5 Acked 1004 Read 3 Done Sent Done false
20712024-02-06T22:16:27.420Z 6 Acked 1005 Write 8 Done Sent Done false
20722024-02-06T22:16:27.420Z 7 Acked 1006 Write 9 Done Sent Done false
20732024-02-06T22:16:27.420Z 8 Acked 1007 Read 1 Done Sent Done false
20742024-02-06T22:16:27.420Z 9 Acked 1008 Flush 0 Done Sent Done false
20752024-02-06T22:16:27.420Z 10 Acked 1009 Write 3 Done Sent Done false
20762024-02-06T22:16:27.421Z 11 Acked 1010 Read 3 Done Sent Done false
20772024-02-06T22:16:27.421Z 12 Acked 1011 Write 6 Done Sent Done false
20782024-02-06T22:16:27.421Z 13 Acked 1012 Read 1 Done Sent Done false
20792024-02-06T22:16:27.421Z 14 Acked 1013 Write 2 Done Sent Done false
20802024-02-06T22:16:27.421Z 15 Acked 1014 Read 3 Done Sent Done false
20812024-02-06T22:16:27.421Z 16 Acked 1015 Write 5 Done Sent Done false
20822024-02-06T22:16:27.421Z 17 Acked 1016 Read 3 Done Sent Done false
20832024-02-06T22:16:27.421Z 18 Acked 1017 Read 9 Done Sent Done false
20842024-02-06T22:16:27.422Z 19 Acked 1018 Flush 0 Done Sent Done false
20852024-02-06T22:16:27.422Z 20 Acked 1019 Write 7 Done Sent Done false
20862024-02-06T22:16:27.422Z 21 Acked 1020 Read 4 Done Sent Done false
20872024-02-06T22:16:27.422Z 22 Acked 1021 Flush 0 Done Sent Done false
20882024-02-06T22:16:27.422Z 23 Acked 1022 Write 2 Done Sent Done false
20892024-02-06T22:16:27.422Z 24 Acked 1023 Write 4 Done Sent Done false
20902024-02-06T22:16:27.422Z 25 Acked 1024 Flush 0 Done Sent Done false
20912024-02-06T22:16:27.422Z 26 Acked 1025 Write 7 Done Sent Done false
20922024-02-06T22:16:27.422Z 27 Acked 1026 Read 1 Done Sent Done false
20932024-02-06T22:16:27.422Z 28 Acked 1027 Write 7 Done Sent Done false
20942024-02-06T22:16:27.422Z 29 Acked 1028 Read 7 Done Sent Done false
20952024-02-06T22:16:27.422Z 30 Acked 1029 Write 6 Sent Sent Sent false
20962024-02-06T22:16:27.422Z STATES DS:0 DS:1 DS:2 TOTAL
20972024-02-06T22:16:27.423Z New 0 0 0 0
20982024-02-06T22:16:27.423Z Sent 1 28 1 30
20992024-02-06T22:16:27.423Z Done 29 2 29 60
21002024-02-06T22:16:27.570Z Skipped 0 0 0 0
21012024-02-06T22:16:27.570Z Error 0 0 0 0
21022024-02-06T22:16:27.570ZLast Flush: 1024 0 1024
21032024-02-06T22:16:27.570ZDownstairs last five completed:
21042024-02-06T22:16:27.571ZUpstairs last five completed: 30 29 28 27 26
21052024-02-06T22:16:27.571Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:26.607668251Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1207}
21062024-02-06T22:16:27.571ZWrote out file "/tmp/test_repair_verify.data"
21072024-02-06T22:16:27.571Z
21082024-02-06T22:16:27.571ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21092024-02-06T22:16:27.571ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21102024-02-06T22:16:27.571Z 0 000-019 10 10 10 20 20 20 F F F
21112024-02-06T22:16:27.571Z 1 020-039 18 12 18 39 24 39 F F F <---
21122024-02-06T22:16:27.571Z 2 040-059 16 16 16 30 30 30 F F F
21132024-02-06T22:16:27.571Z 3 060-079 16 16 16 30 30 30 F F F
21142024-02-06T22:16:27.571Z 4 080-099 18 16 18 38 32 38 F F F <---
21152024-02-06T22:16:27.571Z 5 100-119 18 16 18 37 32 37 F F F <---
21162024-02-06T22:16:27.571Z 6 120-139 18 16 18 40 30 40 F F F <---
21172024-02-06T22:16:27.571Z 7 140-159 18 14 18 40 28 40 F F F <---
21182024-02-06T22:16:27.571Z 8 160-179 14 14 14 28 28 28 F F F
21192024-02-06T22:16:27.571Z 9 180-199 14 14 14 28 28 28 F F F
21202024-02-06T22:16:27.571Z 10 200-219 15 15 15 29 29 29 F F F
21212024-02-06T22:16:27.571Z 11 220-239 15 15 15 29 29 29 F F F
21222024-02-06T22:16:27.571Z 12 240-259 18 16 18 38 30 38 F F F <---
21232024-02-06T22:16:27.572Z 13 260-279 10 10 10 19 19 19 F F F
21242024-02-06T22:16:27.572Z 14 280-299 11 11 11 23 23 23 F F F
21252024-02-06T22:16:27.572Z 15 300-319 16 16 16 34 34 34 F F F
21262024-02-06T22:16:27.572Z 16 320-339 16 16 16 32 32 32 T F T <---
21272024-02-06T22:16:27.572Z 17 340-359 14 14 14 28 28 28 T F T <---
21282024-02-06T22:16:27.572Z 18 360-379 18 17 18 37 36 37 F F F <---
21292024-02-06T22:16:27.572Z 19 380-399 18 13 18 37 25 37 F T F <---
21302024-02-06T22:16:27.572Z 20 400-419 18 14 18 37 27 37 F T F <---
21312024-02-06T22:16:27.572Z 21 420-439 14 14 14 28 28 28 F F F
21322024-02-06T22:16:27.572Z 22 440-459 13 13 13 25 25 25 T F T <---
21332024-02-06T22:16:27.572Z 23 460-479 18 17 18 37 36 37 F F F <---
21342024-02-06T22:16:27.572Z 24 480-499 13 13 13 25 25 25 F F F
21352024-02-06T22:16:27.572Z 25 500-519 17 17 17 36 36 36 F F F
21362024-02-06T22:16:27.572Z 26 520-539 18 13 18 38 25 38 F F F <---
21372024-02-06T22:16:27.572Z 27 540-559 16 16 16 31 31 31 F F F
21382024-02-06T22:16:27.572Z 28 560-579 18 16 18 38 35 38 F F F <---
21392024-02-06T22:16:27.572Z 29 580-599 18 16 18 37 35 37 F F F <---
21402024-02-06T22:16:27.572ZMax gen: 18, Max flush: 40
21412024-02-06T22:16:27.572ZError: Difference in extent metadata found!
21422024-02-06T22:16:27.572ZA Difference in extent metadata is expected here
21432024-02-06T22:16:27.572ZOn loop 9
21442024-02-06T22:16:27.572Z
21452024-02-06T22:16:27.572ZVerifying data now
21462024-02-06T22:16:33.214ZNo change after verify
21472024-02-06T22:16:33.230ZLoop: 9 Downstairs dump after verify (and repair):
21482024-02-06T22:16:33.230ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21492024-02-06T22:16:33.231Z 0 000-019 10 10 10 20 20 20 F F F
21502024-02-06T22:16:33.231Z 1 020-039 18 18 18 39 39 39 F F F
21512024-02-06T22:16:33.231Z 2 040-059 16 16 16 30 30 30 F F F
21522024-02-06T22:16:33.231Z 3 060-079 16 16 16 30 30 30 F F F
21532024-02-06T22:16:33.231Z 4 080-099 18 18 18 38 38 38 F F F
21542024-02-06T22:16:33.231Z 5 100-119 18 18 18 37 37 37 F F F
21552024-02-06T22:16:33.231Z 6 120-139 18 18 18 40 40 40 F F F
21562024-02-06T22:16:33.231Z 7 140-159 18 18 18 40 40 40 F F F
21572024-02-06T22:16:33.231Z 8 160-179 14 14 14 28 28 28 F F F
21582024-02-06T22:16:33.231Z 9 180-199 14 14 14 28 28 28 F F F
21592024-02-06T22:16:33.231Z 10 200-219 15 15 15 29 29 29 F F F
21602024-02-06T22:16:33.231Z 11 220-239 15 15 15 29 29 29 F F F
21612024-02-06T22:16:33.231Z 12 240-259 18 18 18 38 38 38 F F F
21622024-02-06T22:16:33.231Z 13 260-279 10 10 10 19 19 19 F F F
21632024-02-06T22:16:33.232Z 14 280-299 11 11 11 23 23 23 F F F
21642024-02-06T22:16:33.232Z 15 300-319 16 16 16 34 34 34 F F F
21652024-02-06T22:16:33.232Z 16 320-339 19 19 19 41 41 41 F F F
21662024-02-06T22:16:33.232Z 17 340-359 19 19 19 41 41 41 F F F
21672024-02-06T22:16:33.232Z 18 360-379 18 18 18 37 37 37 F F F
21682024-02-06T22:16:33.232Z 19 380-399 18 18 18 37 37 37 F F F
21692024-02-06T22:16:33.232Z 20 400-419 18 18 18 37 37 37 F F F
21702024-02-06T22:16:33.232Z 21 420-439 14 14 14 28 28 28 F F F
21712024-02-06T22:16:33.232Z 22 440-459 19 19 19 41 41 41 F F F
21722024-02-06T22:16:33.232Z 23 460-479 18 18 18 37 37 37 F F F
21732024-02-06T22:16:33.232Z 24 480-499 13 13 13 25 25 25 F F F
21742024-02-06T22:16:33.232Z 25 500-519 17 17 17 36 36 36 F F F
21752024-02-06T22:16:33.232Z 26 520-539 18 18 18 38 38 38 F F F
21762024-02-06T22:16:33.232Z 27 540-559 16 16 16 31 31 31 F F F
21772024-02-06T22:16:33.232Z 28 560-579 18 18 18 38 38 38 F F F
21782024-02-06T22:16:33.232Z 29 580-599 18 18 18 37 37 37 F F F
21792024-02-06T22:16:33.232ZMax gen: 19, Max flush: 41
21802024-02-06T22:16:33.233Z
21812024-02-06T22:16:33.233ZBegin loop 10
21822024-02-06T22:16:33.233ZDownstairs to restart: 0
21832024-02-06T22:16:33.233Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.43269221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215}
21842024-02-06T22:16:33.233Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
21852024-02-06T22:16:33.233Z,"time":"2024-02-06T22:16:32.433010157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215}
21862024-02-06T22:16:33.233Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433041072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215}
21872024-02-06T22:16:33.233Z{"msg":"Crucible 271073a8-c7b3-4062-8882-5f3030c53b6e has session id: fc8ce3a0-b646-483a-b11f-91cd7a799a1b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433076618Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21882024-02-06T22:16:33.233Z{"msg":"Upstairs opts: Upstairs UUID: 271073a8-c7b3-4062-8882-5f3030c53b6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433125391Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21892024-02-06T22:16:33.233Z{"msg":"Crucible stats registered with UUID: 271073a8-c7b3-4062-8882-5f3030c53b6e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433151848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21902024-02-06T22:16:33.233Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433186243Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215}
21912024-02-06T22:16:33.233Z{"msg":"271073a8-c7b3-4062-8882-5f3030c53b6e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.43321006Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21922024-02-06T22:16:33.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433243285Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21932024-02-06T22:16:33.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433269892Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21942024-02-06T22:16:33.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433306287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21952024-02-06T22:16:33.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433335743Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21962024-02-06T22:16:33.234Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433374658Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21972024-02-06T22:16:33.234Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433447628Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21982024-02-06T22:16:33.234Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433516558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
21992024-02-06T22:16:33.234Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433613185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22002024-02-06T22:16:33.234Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433661839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22012024-02-06T22:16:33.234Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433691865Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22022024-02-06T22:16:33.234Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:32.433744948Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22032024-02-06T22:16:33.234Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:32.433776253Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22042024-02-06T22:16:33.234Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433791611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22052024-02-06T22:16:33.234Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:32.433848734Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22062024-02-06T22:16:33.234Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433909085Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22072024-02-06T22:16:33.234Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433955029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22082024-02-06T22:16:33.234Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.433987965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22092024-02-06T22:16:33.234Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.434278325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22102024-02-06T22:16:33.234Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.434326819Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22112024-02-06T22:16:33.234Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.434377112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22122024-02-06T22:16:33.235Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.434407568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22132024-02-06T22:16:33.235Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.434933047Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22142024-02-06T22:16:33.235Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.434978381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22152024-02-06T22:16:33.235Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.435510979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22162024-02-06T22:16:33.235Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.435552183Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22172024-02-06T22:16:33.235Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.435570421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22182024-02-06T22:16:33.235Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.435599257Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22192024-02-06T22:16:33.235Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.435613975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22202024-02-06T22:16:33.235Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:32.435627603Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22212024-02-06T22:16:34.208Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.434357543Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22222024-02-06T22:16:34.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.434571544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22232024-02-06T22:16:34.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.435245563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22242024-02-06T22:16:34.213Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.435317453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22252024-02-06T22:16:34.213Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.435763053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22262024-02-06T22:16:34.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436236449Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22272024-02-06T22:16:34.213Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436296771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22282024-02-06T22:16:34.214Z{"msg":"[0]R flush_numbers[0..12]: [20, 39, 30, 30, 38, 37, 40, 40, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436328527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22292024-02-06T22:16:34.214Z{"msg":"[0]R generation[0..12]: [10, 18, 16, 16, 18, 18, 18, 18, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436358292Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22302024-02-06T22:16:34.214Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436381269Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22312024-02-06T22:16:34.214ZWait for a query_work_queue command to finish before sending IO
22322024-02-06T22:16:34.214Z{"msg":"[1]R flush_numbers[0..12]: [20, 39, 30, 30, 38, 37, 40, 40, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436431802Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22332024-02-06T22:16:34.214Z{"msg":"[1]R generation[0..12]: [10, 18, 16, 16, 18, 18, 18, 18, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436478736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22342024-02-06T22:16:34.214Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436499933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22352024-02-06T22:16:34.214Z{"msg":"[2]R flush_numbers[0..12]: [20, 39, 30, 30, 38, 37, 40, 40, 28, 28, 29, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436531639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22362024-02-06T22:16:34.214Z{"msg":"[2]R generation[0..12]: [10, 18, 16, 16, 18, 18, 18, 18, 14, 14, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436577273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22372024-02-06T22:16:34.214Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436616637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22382024-02-06T22:16:34.214Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436633305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22392024-02-06T22:16:34.214Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436661871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22402024-02-06T22:16:34.215Z{"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436698377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22412024-02-06T22:16:34.215Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436727473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22422024-02-06T22:16:34.215Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.43674507Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22432024-02-06T22:16:34.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436778486Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22442024-02-06T22:16:34.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436811601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22452024-02-06T22:16:34.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436827719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22462024-02-06T22:16:34.215Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436859285Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22472024-02-06T22:16:34.215Z}
22482024-02-06T22:16:34.215Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436898859Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22492024-02-06T22:16:34.215Z{"msg":"271073a8-c7b3-4062-8882-5f3030c53b6e is now active with session: fc8ce3a0-b646-483a-b11f-91cd7a799a1b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436932625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22502024-02-06T22:16:34.215Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436963371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215,"session_id":"fc8ce3a0-b646-483a-b11f-91cd7a799a1b"}
22512024-02-06T22:16:34.215Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:33.436996046Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215}
22522024-02-06T22:16:34.215ZLoading write count information from file "/tmp/test_repair_verify.data"
22532024-02-06T22:16:34.216ZRun Repair workload
22542024-02-06T22:16:34.216Z01/30 Write block 511 len 4096 data: 2
22552024-02-06T22:16:34.216Z02/30 Read block 78 len 32768
22562024-02-06T22:16:34.216Z03/30 Read block 60 len 8192
22572024-02-06T22:16:34.216Z04/30 Write block 335 len 4096 data: 3
22582024-02-06T22:16:34.216Z05/30 Read block 94 len 12288
22592024-02-06T22:16:34.216Z06/30 Write block 15 len 8192 data: 3 3
22602024-02-06T22:16:34.216Z07/30 Write block 203 len 16384 data: 4 4 4 4
22612024-02-06T22:16:34.216Z08/30 Read block 92 len 4096
22622024-02-06T22:16:34.216Z09/30 Write block 240 len 16384 data: 3 3 4 4
22632024-02-06T22:16:34.216Z10/30 Flush
22642024-02-06T22:16:34.216Z11/30 Write block 163 len 28672 data: 2 2 2 2 2 2 2
22652024-02-06T22:16:34.216Z12/30 Read block 499 len 40960
22662024-02-06T22:16:34.217Z13/30 Read block 139 len 12288
22672024-02-06T22:16:34.217Z14/30 Flush
22682024-02-06T22:16:34.220Z15/30 Write block 40 len 4096 data: 2
22692024-02-06T22:16:34.220Z16/30 Write block 503 len 24576 data: 4 2 2 2 2 2
22702024-02-06T22:16:34.220Z17/30 Write block 308 len 16384 data: 2 2 3 3
22712024-02-06T22:16:34.220Z18/30 Write block 172 len 8192 data: 2 2
22722024-02-06T22:16:34.220Z19/30 Write block 513 len 4096 data: 4
22732024-02-06T22:16:34.221Z20/30 Flush
22742024-02-06T22:16:34.221Z21/30 Write block 491 len 40960 data: 2 2 2 3 3 3 4 4 4 4
22752024-02-06T22:16:34.221Z22/30 Read block 281 len 32768
22762024-02-06T22:16:34.221Z23/30 Read block 347 len 20480
22772024-02-06T22:16:34.226Z24/30 Read block 104 len 12288
22782024-02-06T22:16:34.226Z25/30 Write block 102 len 16384 data: 2 2 2 2
22792024-02-06T22:16:34.226Z26/30 Read block 68 len 32768
22802024-02-06T22:16:34.226Z27/30 Write block 22 len 36864 data: 2 2 2 3 4 4 4 3 3
22812024-02-06T22:16:34.226Z28/30 Write block 541 len 40960 data: 2 2 2 2 2 2 2 2 3 2
22822024-02-06T22:16:34.227Z29/30 Read block 335 len 20480
22832024-02-06T22:16:34.231Z30/30 Write block 197 len 4096 data: 2
22842024-02-06T22:16:34.231Z----------------------------------------------------------------
22852024-02-06T22:16:34.231Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22862024-02-06T22:16:34.231ZStates: Active Active Active
22872024-02-06T22:16:34.231ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22882024-02-06T22:16:34.231Z 1 Acked 1000 Write 1 Done Done Done false
22892024-02-06T22:16:34.231Z 2 Acked 1001 Read 8 Sent Done Done false
22902024-02-06T22:16:34.231Z 3 Acked 1002 Read 2 Sent Done Done false
22912024-02-06T22:16:34.231Z 4 Acked 1003 Write 1 Sent Done Done false
22922024-02-06T22:16:34.231Z 5 Acked 1004 Read 3 Sent Done Done false
22932024-02-06T22:16:34.232Z 6 Acked 1005 Write 2 Sent Done Done false
22942024-02-06T22:16:34.232Z 7 Acked 1006 Write 4 Sent Done Done false
22952024-02-06T22:16:34.232Z 8 Acked 1007 Read 1 Sent Done Done false
22962024-02-06T22:16:34.232Z 9 Acked 1008 Write 4 Sent Done Done false
22972024-02-06T22:16:34.232Z 10 Acked 1009 Flush 0 Sent Done Done false
22982024-02-06T22:16:34.232Z 11 Acked 1010 Write 7 Sent Done Done false
22992024-02-06T22:16:34.232Z 12 Acked 1011 Read 10 Sent Done Done false
23002024-02-06T22:16:34.232Z 13 Acked 1012 Read 3 Sent Done Done false
23012024-02-06T22:16:34.232Z 14 Acked 1013 Flush 0 Sent Done Done false
23022024-02-06T22:16:34.232Z 15 Acked 1014 Write 1 Sent Done Done false
23032024-02-06T22:16:34.232Z 16 Acked 1015 Write 6 Sent Done Done false
23042024-02-06T22:16:34.232Z 17 Acked 1016 Write 4 Sent Done Done false
23052024-02-06T22:16:34.232Z 18 Acked 1017 Write 2 Sent Done Done false
23062024-02-06T22:16:34.232Z 19 Acked 1018 Write 1 Sent Done Done false
23072024-02-06T22:16:34.233Z 20 Acked 1019 Flush 0 Sent Done Done false
23082024-02-06T22:16:34.233Z 21 Acked 1020 Write 10 Sent Done Done false
23092024-02-06T22:16:34.233Z 22 Acked 1021 Read 8 Sent Done Done false
23102024-02-06T22:16:34.233Z 23 Acked 1022 Read 5 Sent Done Done false
23112024-02-06T22:16:34.233Z 24 Acked 1023 Read 3 Sent Done Done false
23122024-02-06T22:16:34.233Z 25 Acked 1024 Write 4 Sent Done Done false
23132024-02-06T22:16:34.233Z 26 Acked 1025 Read 8 Sent Done Done false
23142024-02-06T22:16:34.233Z 27 Acked 1026 Write 9 Sent Done Done false
23152024-02-06T22:16:34.233Z 28 Acked 1027 Write 10 Sent Done Done false
23162024-02-06T22:16:34.233Z 29 Acked 1028 Read 5 Sent Done Done false
23172024-02-06T22:16:34.233Z 30 Acked 1029 Write 1 Sent Sent Sent false
23182024-02-06T22:16:34.233Z STATES DS:0 DS:1 DS:2 TOTAL
23192024-02-06T22:16:34.233Z New 0 0 0 0
23202024-02-06T22:16:34.233Z Sent 29 1 1 31
23212024-02-06T22:16:34.233Z Done 1 29 29 59
23222024-02-06T22:16:34.233Z Skipped 0 0 0 0
23232024-02-06T22:16:34.233Z Error 0 0 0 0
23242024-02-06T22:16:34.233ZLast Flush: 0 1019 1019
23252024-02-06T22:16:34.233ZDownstairs last five completed:
23262024-02-06T22:16:34.233ZUpstairs last five completed: 30 29 28 27 26
23272024-02-06T22:16:34.234Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:33.466262892Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1215}
23282024-02-06T22:16:34.234ZWrote out file "/tmp/test_repair_verify.data"
23292024-02-06T22:16:34.237Z
23302024-02-06T22:16:34.237ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23312024-02-06T22:16:34.253ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23322024-02-06T22:16:34.253Z 0 000-019 10 20 20 20 42 42 F F F <---
23332024-02-06T22:16:34.253Z 1 020-039 18 18 18 39 39 39 F T T <---
23342024-02-06T22:16:34.253Z 2 040-059 16 20 20 30 44 44 F F F <---
23352024-02-06T22:16:34.253Z 3 060-079 16 16 16 30 30 30 F F F
23362024-02-06T22:16:34.253Z 4 080-099 18 18 18 38 38 38 F F F
23372024-02-06T22:16:34.253Z 5 100-119 18 18 18 37 37 37 F T T <---
23382024-02-06T22:16:34.257Z 6 120-139 18 18 18 40 40 40 F F F
23392024-02-06T22:16:34.257Z 7 140-159 18 18 18 40 40 40 F F F
23402024-02-06T22:16:34.257Z 8 160-179 14 20 20 28 44 44 F F F <---
23412024-02-06T22:16:34.257Z 9 180-199 14 14 14 28 28 28 F T T <---
23422024-02-06T22:16:34.257Z 10 200-219 15 20 20 29 42 42 F F F <---
23432024-02-06T22:16:34.257Z 11 220-239 15 15 15 29 29 29 F F F
23442024-02-06T22:16:34.257Z 12 240-259 18 20 20 38 42 42 F F F <---
23452024-02-06T22:16:34.257Z 13 260-279 10 10 10 19 19 19 F F F
23462024-02-06T22:16:34.257Z 14 280-299 11 11 11 23 23 23 F F F
23472024-02-06T22:16:34.257Z 15 300-319 16 20 20 34 44 44 F F F <---
23482024-02-06T22:16:34.257Z 16 320-339 19 20 20 41 42 42 F F F <---
23492024-02-06T22:16:34.258Z 17 340-359 19 19 19 41 41 41 F F F
23502024-02-06T22:16:34.258Z 18 360-379 18 18 18 37 37 37 F F F
23512024-02-06T22:16:34.258Z 19 380-399 18 18 18 37 37 37 F F F
23522024-02-06T22:16:34.258Z 20 400-419 18 18 18 37 37 37 F F F
23532024-02-06T22:16:34.258Z 21 420-439 14 14 14 28 28 28 F F F
23542024-02-06T22:16:34.258Z 22 440-459 19 19 19 41 41 41 F F F
23552024-02-06T22:16:34.258Z 23 460-479 18 18 18 37 37 37 F F F
23562024-02-06T22:16:34.258Z 24 480-499 13 13 13 25 25 25 F T T <---
23572024-02-06T22:16:34.258Z 25 500-519 17 20 20 36 44 44 T T T <---
23582024-02-06T22:16:34.258Z 26 520-539 18 18 18 38 38 38 F F F
23592024-02-06T22:16:34.258Z 27 540-559 16 16 16 31 31 31 F T T <---
23602024-02-06T22:16:34.258Z 28 560-579 18 18 18 38 38 38 F F F
23612024-02-06T22:16:34.258Z 29 580-599 18 18 18 37 37 37 F F F
23622024-02-06T22:16:34.258ZMax gen: 20, Max flush: 44
23632024-02-06T22:16:34.259ZError: Difference in extent metadata found!
23642024-02-06T22:16:34.259ZA Difference in extent metadata is expected here
23652024-02-06T22:16:34.259ZOn loop 10
23662024-02-06T22:16:34.259Z
23672024-02-06T22:16:34.262ZVerifying data now
23682024-02-06T22:16:38.903ZNo change after verify
23692024-02-06T22:16:38.906ZLoop: 10 Downstairs dump after verify (and repair):
23702024-02-06T22:16:38.918ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23712024-02-06T22:16:38.919Z 0 000-019 20 20 20 42 42 42 F F F
23722024-02-06T22:16:38.919Z 1 020-039 21 21 21 45 45 45 F F F
23732024-02-06T22:16:38.919Z 2 040-059 20 20 20 44 44 44 F F F
23742024-02-06T22:16:38.919Z 3 060-079 16 16 16 30 30 30 F F F
23752024-02-06T22:16:38.919Z 4 080-099 18 18 18 38 38 38 F F F
23762024-02-06T22:16:38.919Z 5 100-119 21 21 21 45 45 45 F F F
23772024-02-06T22:16:38.919Z 6 120-139 18 18 18 40 40 40 F F F
23782024-02-06T22:16:38.923Z 7 140-159 18 18 18 40 40 40 F F F
23792024-02-06T22:16:38.923Z 8 160-179 20 20 20 44 44 44 F F F
23802024-02-06T22:16:38.923Z 9 180-199 21 21 21 45 45 45 F F F
23812024-02-06T22:16:38.923Z 10 200-219 20 20 20 42 42 42 F F F
23822024-02-06T22:16:38.923Z 11 220-239 15 15 15 29 29 29 F F F
23832024-02-06T22:16:38.923Z 12 240-259 20 20 20 42 42 42 F F F
23842024-02-06T22:16:38.923Z 13 260-279 10 10 10 19 19 19 F F F
23852024-02-06T22:16:38.923Z 14 280-299 11 11 11 23 23 23 F F F
23862024-02-06T22:16:38.923Z 15 300-319 20 20 20 44 44 44 F F F
23872024-02-06T22:16:38.923Z 16 320-339 20 20 20 42 42 42 F F F
23882024-02-06T22:16:38.923Z 17 340-359 19 19 19 41 41 41 F F F
23892024-02-06T22:16:38.923Z 18 360-379 18 18 18 37 37 37 F F F
23902024-02-06T22:16:38.923Z 19 380-399 18 18 18 37 37 37 F F F
23912024-02-06T22:16:38.923Z 20 400-419 18 18 18 37 37 37 F F F
23922024-02-06T22:16:38.924Z 21 420-439 14 14 14 28 28 28 F F F
23932024-02-06T22:16:38.924Z 22 440-459 19 19 19 41 41 41 F F F
23942024-02-06T22:16:38.924Z 23 460-479 18 18 18 37 37 37 F F F
23952024-02-06T22:16:38.924Z 24 480-499 21 21 21 45 45 45 F F F
23962024-02-06T22:16:38.924Z 25 500-519 21 21 21 45 45 45 F F F
23972024-02-06T22:16:38.924Z 26 520-539 18 18 18 38 38 38 F F F
23982024-02-06T22:16:38.924Z 27 540-559 21 21 21 45 45 45 F F F
23992024-02-06T22:16:38.924Z 28 560-579 18 18 18 38 38 38 F F F
24002024-02-06T22:16:38.924Z 29 580-599 18 18 18 37 37 37 F F F
24012024-02-06T22:16:38.924ZMax gen: 21, Max flush: 45
24022024-02-06T22:16:38.924Z
24032024-02-06T22:16:38.924ZBegin loop 11
24042024-02-06T22:16:38.924ZDownstairs to restart: 2
24052024-02-06T22:16:38.940Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.177880316Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223}
24062024-02-06T22:16:38.940ZCrucible runtime is spawned
24072024-02-06T22:16:38.946Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178195123Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223}
24082024-02-06T22:16:38.946Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178234388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223}
24092024-02-06T22:16:38.946Z{"msg":"Crucible 2250f7dc-54d7-4b63-ae90-558a526d308c has session id: 7d738724-6a3a-4a81-a9fb-edbb3d9db1bb","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178258964Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24102024-02-06T22:16:38.946Z{"msg":"Upstairs opts: Upstairs UUID: 2250f7dc-54d7-4b63-ae90-558a526d308c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178306248Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24112024-02-06T22:16:38.946Z{"msg":"Crucible stats registered with UUID: 2250f7dc-54d7-4b63-ae90-558a526d308c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178342963Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24122024-02-06T22:16:38.946Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.17836775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223}
24132024-02-06T22:16:38.946Z{"msg":"2250f7dc-54d7-4b63-ae90-558a526d308c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178392106Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24142024-02-06T22:16:38.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178435491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24152024-02-06T22:16:38.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178479435Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24162024-02-06T22:16:38.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178519779Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24172024-02-06T22:16:38.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178546196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24182024-02-06T22:16:38.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178561344Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24192024-02-06T22:16:38.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178580201Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24202024-02-06T22:16:38.946Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178621506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24212024-02-06T22:16:38.946Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178656111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24222024-02-06T22:16:38.946Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178685227Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24232024-02-06T22:16:38.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178724521Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24242024-02-06T22:16:38.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178752078Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24252024-02-06T22:16:38.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.178870272Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24262024-02-06T22:16:38.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.179458612Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24272024-02-06T22:16:38.947Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.179503856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24282024-02-06T22:16:38.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.179534832Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24292024-02-06T22:16:38.947Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.179582396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24302024-02-06T22:16:38.947Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180134911Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24312024-02-06T22:16:38.947Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180265203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24322024-02-06T22:16:38.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180307908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24332024-02-06T22:16:38.947Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180334934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24342024-02-06T22:16:38.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180605218Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24352024-02-06T22:16:38.947Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.18066313Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24362024-02-06T22:16:38.947Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180707744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24372024-02-06T22:16:38.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180861573Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24382024-02-06T22:16:38.947Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180913746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24392024-02-06T22:16:38.947Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.18095913Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24402024-02-06T22:16:38.947Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.180990206Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24412024-02-06T22:16:38.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181375983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24422024-02-06T22:16:38.947Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181421107Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24432024-02-06T22:16:38.947Z{"msg":"[0]R flush_numbers[0..12]: [42, 45, 44, 30, 38, 45, 40, 40, 44, 45, 42, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181457892Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24442024-02-06T22:16:38.947Z{"msg":"[0]R generation[0..12]: [20, 21, 20, 16, 18, 21, 18, 18, 20, 21, 20, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181495037Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24452024-02-06T22:16:38.947Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181531262Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24462024-02-06T22:16:38.947Z{"msg":"[1]R flush_numbers[0..12]: [42, 45, 44, 30, 38, 45, 40, 40, 44, 45, 42, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181568347Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24472024-02-06T22:16:38.947Z{"msg":"[1]R generation[0..12]: [20, 21, 20, 16, 18, 21, 18, 18, 20, 21, 20, 15]"Wait for a query_work_queue command to finish before sending IO
24482024-02-06T22:16:38.947Z,"v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181610712Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24492024-02-06T22:16:38.947Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181652666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24502024-02-06T22:16:38.948Z{"msg":"[2]R flush_numbers[0..12]: [42, 45, 44, 30, 38, 45, 40, 40, 44, 45, 42, 29]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181690931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24512024-02-06T22:16:38.951Z{"msg":"[2]R generation[0..12]: [20, 21, 20, 16, 18, 21, 18, 18, 20, 21, 20, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181727726Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24522024-02-06T22:16:38.951Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181802046Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24532024-02-06T22:16:38.951Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.18184296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24542024-02-06T22:16:38.951Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181880805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24552024-02-06T22:16:38.951Z{"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.18191778Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24562024-02-06T22:16:38.952Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.181954455Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24572024-02-06T22:16:38.952Z7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24582024-02-06T22:16:38.952Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.182007188Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24592024-02-06T22:16:38.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.182032375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24602024-02-06T22:16:38.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.182047672Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24612024-02-06T22:16:38.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.18206823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24622024-02-06T22:16:38.952Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.182082278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24632024-02-06T22:16:38.952Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.182101575Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24642024-02-06T22:16:38.952Z{"msg":"2250f7dc-54d7-4b63-ae90-558a526d308c is now active with session: 7d738724-6a3a-4a81-a9fb-edbb3d9db1bb","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.182115054Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24652024-02-06T22:16:38.952Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.182127702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223,"session_id":"7d738724-6a3a-4a81-a9fb-edbb3d9db1bb"}
24662024-02-06T22:16:38.952Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:38.18214014Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223}
24672024-02-06T22:16:38.952ZLoading write count information from file "/tmp/test_repair_verify.data"
24682024-02-06T22:16:38.952ZRun Repair workload
24692024-02-06T22:16:38.952Z01/30 Flush
24702024-02-06T22:16:38.952Z02/30 Write block 391 len 28672 data: 3 3 2 2 3 3 3
24712024-02-06T22:16:38.952Z03/30 Write block 219 len 16384 data: 3 4 4 3
24722024-02-06T22:16:38.952Z04/30 Write block 3 len 32768 data: 2 2 2 2 2 2 2 2
24732024-02-06T22:16:38.952Z05/30 Write block 489 len 32768 data: 2 2 3 3 3 4 4 4
24742024-02-06T22:16:38.952Z06/30 Read block 171 len 28672
24752024-02-06T22:16:38.952Z07/30 Read block 70 len 12288
24762024-02-06T22:16:38.955Z08/30 Read block 373 len 20480
24772024-02-06T22:16:38.956Z09/30 Write block 143 len 28672 data: 4 4 5 5 6 8 7
24782024-02-06T22:16:38.956Z10/30 Flush
24792024-02-06T22:16:38.959Z11/30 Write block 592 len 28672 data: 5 5 4 4 4 4 4
24802024-02-06T22:16:38.959Z12/30 Write block 125 len 28672 data: 6 6 6 6 4 4 4
24812024-02-06T22:16:38.959Z13/30 Read block 166 len 16384
24822024-02-06T22:16:38.962Z14/30 Read block 87 len 24576
24832024-02-06T22:16:38.962Z15/30 Write block 428 len 12288 data: 3 3 3
24842024-02-06T22:16:38.962Z16/30 Flush
24852024-02-06T22:16:38.968Z17/30 Write block 376 len 8192 data: 5 6
24862024-02-06T22:16:38.968Z18/30 Write block 476 len 20480 data: 5 5 5 4 2
24872024-02-06T22:16:38.968Z19/30 Read block 91 len 24576
24882024-02-06T22:16:38.968Z20/30 Write block 417 len 24576 data: 4 4 3 3 3 2
24892024-02-06T22:16:38.968Z21/30 Write block 357 len 24576 data: 5 5 5 5 5 5
24902024-02-06T22:16:38.968Z22/30 Read block 378 len 12288
24912024-02-06T22:16:38.971Z23/30 Write block 501 len 40960 data: 4 4 5 3 3 3 3 3 2 2
24922024-02-06T22:16:38.971Z24/30 Read block 593 len 28672
24932024-02-06T22:16:38.972Z25/30 Write block 95 len 24576 data: 2 2 2 3 3 3
24942024-02-06T22:16:38.972Z26/30 Write block 430 len 4096 data: 4
24952024-02-06T22:16:38.972Z27/30 Read block 364 len 16384
24962024-02-06T22:16:38.975Z28/30 Write block 377 len 40960 data: 7 6 5 5 4 3 3 3 3 3
24972024-02-06T22:16:38.975Z29/30 Read block 153 len 40960
24982024-02-06T22:16:38.978Z30/30 Read block 133 len 40960
24992024-02-06T22:16:38.978Z----------------------------------------------------------------
25002024-02-06T22:16:38.978Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:20
25012024-02-06T22:16:38.978ZStates: Active Active Active
25022024-02-06T22:16:38.978ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
25032024-02-06T22:16:38.978Z 11 Acked 1010 Write 7 Done Done Done false
25042024-02-06T22:16:38.978Z 12 Acked 1011 Write 7 Done Done Done false
25052024-02-06T22:16:38.978Z 13 Acked 1012 Read 4 Done Done Done false
25062024-02-06T22:16:38.978Z 14 Acked 1013 Read 6 Done Done Sent false
25072024-02-06T22:16:38.978Z 15 Acked 1014 Write 3 Done Done Sent false
25082024-02-06T22:16:38.978Z 16 Acked 1015 Flush 0 Done Done Sent false
25092024-02-06T22:16:38.978Z 17 Acked 1016 Write 2 Done Done Sent false
25102024-02-06T22:16:38.978Z 18 Acked 1017 Write 5 Done Done Sent false
25112024-02-06T22:16:38.978Z 19 Acked 1018 Read 6 Done Done Sent false
25122024-02-06T22:16:38.978Z 20 Acked 1019 Write 6 Done Done Sent false
25132024-02-06T22:16:38.978Z 21 Acked 1020 Write 6 Done Done Sent false
25142024-02-06T22:16:38.978Z 22 Acked 1021 Read 3 Done Done Sent false
25152024-02-06T22:16:38.979Z 23 Acked 1022 Write 10 Done Done Sent false
25162024-02-06T22:16:38.979Z 24 Acked 1023 Read 7 Done Done Sent false
25172024-02-06T22:16:38.979Z 25 Acked 1024 Write 6 Done Done Sent false
25182024-02-06T22:16:38.979Z 26 Acked 1025 Write 1 Done Done Sent false
25192024-02-06T22:16:38.979Z 27 Acked 1026 Read 4 Done Done Sent false
25202024-02-06T22:16:38.979Z 28 Acked 1027 Write 10 Done Done Sent false
25212024-02-06T22:16:38.979Z 29 Acked 1028 Read 10 Done Done Sent false
25222024-02-06T22:16:38.979Z 30 Acked 1029 Read 10 Done Sent Sent false
25232024-02-06T22:16:38.979Z STATES DS:0 DS:1 DS:2 TOTAL
25242024-02-06T22:16:38.979Z New 0 0 0 0
25252024-02-06T22:16:38.979Z Sent 0 1 17 18
25262024-02-06T22:16:38.979Z Done 20 19 3 42
25272024-02-06T22:16:38.979Z Skipped 0 0 0 0
25282024-02-06T22:16:38.979Z Error 0 0 0 0
25292024-02-06T22:16:38.979ZLast Flush: 1015 1015 1009
25302024-02-06T22:16:38.979ZDownstairs last five completed: 1009 1008 1007 1006 1005
25312024-02-06T22:16:38.979ZUpstairs last five completed: 30 29 28 27 26
25322024-02-06T22:16:38.979Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:38.214795998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1223}
25332024-02-06T22:16:38.979ZWrote out file "/tmp/test_repair_verify.data"
25342024-02-06T22:16:38.982Z
25352024-02-06T22:16:38.985ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25362024-02-06T22:16:39.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25372024-02-06T22:16:39.006Z 0 000-019 22 22 22 47 47 47 F F F
25382024-02-06T22:16:39.006Z 1 020-039 21 21 21 45 45 45 F F F
25392024-02-06T22:16:39.006Z 2 040-059 20 20 20 44 44 44 F F F
25402024-02-06T22:16:39.006Z 3 060-079 16 16 16 30 30 30 F F F
25412024-02-06T22:16:39.006Z 4 080-099 18 18 18 38 38 38 T T F <---
25422024-02-06T22:16:39.006Z 5 100-119 21 21 21 45 45 45 T T F <---
25432024-02-06T22:16:39.010Z 6 120-139 22 22 18 48 48 40 F F T <---
25442024-02-06T22:16:39.010Z 7 140-159 22 22 22 47 47 47 F F F
25452024-02-06T22:16:39.010Z 8 160-179 20 20 20 44 44 44 F F F
25462024-02-06T22:16:39.010Z 9 180-199 21 21 21 45 45 45 F F F
25472024-02-06T22:16:39.010Z 10 200-219 22 22 22 47 47 47 F F F
25482024-02-06T22:16:39.010Z 11 220-239 22 22 22 47 47 47 F F F
25492024-02-06T22:16:39.010Z 12 240-259 20 20 20 42 42 42 F F F
25502024-02-06T22:16:39.010Z 13 260-279 10 10 10 19 19 19 F F F
25512024-02-06T22:16:39.010Z 14 280-299 11 11 11 23 23 23 F F F
25522024-02-06T22:16:39.010Z 15 300-319 20 20 20 44 44 44 F F F
25532024-02-06T22:16:39.010Z 16 320-339 20 20 20 42 42 42 F F F
25542024-02-06T22:16:39.010Z 17 340-359 19 19 19 41 41 41 T T F <---
25552024-02-06T22:16:39.010Z 18 360-379 18 18 18 37 37 37 T T F <---
25562024-02-06T22:16:39.010Z 19 380-399 22 22 22 47 47 47 T T F <---
25572024-02-06T22:16:39.010Z 20 400-419 18 18 18 37 37 37 T T F <---
25582024-02-06T22:16:39.010Z 21 420-439 22 22 14 48 48 28 T T F <---
25592024-02-06T22:16:39.010Z 22 440-459 19 19 19 41 41 41 F F F
25602024-02-06T22:16:39.010Z 23 460-479 18 18 18 37 37 37 T T F <---
25612024-02-06T22:16:39.011Z 24 480-499 22 22 22 47 47 47 T T F <---
25622024-02-06T22:16:39.011Z 25 500-519 21 21 21 45 45 45 T T F <---
25632024-02-06T22:16:39.011Z 26 520-539 18 18 18 38 38 38 F F F
25642024-02-06T22:16:39.011Z 27 540-559 21 21 21 45 45 45 F F F
25652024-02-06T22:16:39.011Z 28 560-579 18 18 18 38 38 38 F F F
25662024-02-06T22:16:39.011Z 29 580-599 22 22 18 48 48 37 F F T <---
25672024-02-06T22:16:39.011ZMax gen: 22, Max flush: 48
25682024-02-06T22:16:39.011ZError: Difference in extent metadata found!
25692024-02-06T22:16:39.011ZA Difference in extent metadata is expected here
25702024-02-06T22:16:39.011ZOn loop 11
25712024-02-06T22:16:39.011Z
25722024-02-06T22:16:39.011ZVerifying data now
25732024-02-06T22:16:44.658ZNo change after verify
25742024-02-06T22:16:44.659ZLoop: 11 Downstairs dump after verify (and repair):
25752024-02-06T22:16:44.664ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25762024-02-06T22:16:44.664Z 0 000-019 22 22 22 47 47 47 F F F
25772024-02-06T22:16:44.664Z 1 020-039 21 21 21 45 45 45 F F F
25782024-02-06T22:16:44.664Z 2 040-059 20 20 20 44 44 44 F F F
25792024-02-06T22:16:44.664Z 3 060-079 16 16 16 30 30 30 F F F
25802024-02-06T22:16:44.665Z 4 080-099 23 23 23 49 49 49 F F F
25812024-02-06T22:16:44.665Z 5 100-119 23 23 23 49 49 49 F F F
25822024-02-06T22:16:44.665Z 6 120-139 22 22 22 48 48 48 F F F
25832024-02-06T22:16:44.665Z 7 140-159 22 22 22 47 47 47 F F F
25842024-02-06T22:16:44.665Z 8 160-179 20 20 20 44 44 44 F F F
25852024-02-06T22:16:44.665Z 9 180-199 21 21 21 45 45 45 F F F
25862024-02-06T22:16:44.665Z 10 200-219 22 22 22 47 47 47 F F F
25872024-02-06T22:16:44.665Z 11 220-239 22 22 22 47 47 47 F F F
25882024-02-06T22:16:44.666Z 12 240-259 20 20 20 42 42 42 F F F
25892024-02-06T22:16:44.666Z 13 260-279 10 10 10 19 19 19 F F F
25902024-02-06T22:16:44.666Z 14 280-299 11 11 11 23 23 23 F F F
25912024-02-06T22:16:44.666Z 15 300-319 20 20 20 44 44 44 F F F
25922024-02-06T22:16:44.666Z 16 320-339 20 20 20 42 42 42 F F F
25932024-02-06T22:16:44.666Z 17 340-359 23 23 23 49 49 49 F F F
25942024-02-06T22:16:44.666Z 18 360-379 23 23 23 49 49 49 F F F
25952024-02-06T22:16:44.667Z 19 380-399 23 23 23 49 49 49 F F F
25962024-02-06T22:16:44.667Z 20 400-419 23 23 23 49 49 49 F F F
25972024-02-06T22:16:44.667Z 21 420-439 23 23 23 49 49 49 F F F
25982024-02-06T22:16:44.667Z 22 440-459 19 19 19 41 41 41 F F F
25992024-02-06T22:16:44.667Z 23 460-479 23 23 23 49 49 49 F F F
26002024-02-06T22:16:44.667Z 24 480-499 23 23 23 49 49 49 F F F
26012024-02-06T22:16:44.667Z 25 500-519 23 23 23 49 49 49 F F F
26022024-02-06T22:16:44.668Z 26 520-539 18 18 18 38 38 38 F F F
26032024-02-06T22:16:44.668Z 27 540-559 21 21 21 45 45 45 F F F
26042024-02-06T22:16:44.668Z 28 560-579 18 18 18 38 38 38 F F F
26052024-02-06T22:16:44.668Z 29 580-599 22 22 22 48 48 48 F F F
26062024-02-06T22:16:44.668ZMax gen: 23, Max flush: 49
26072024-02-06T22:16:44.668Z
26082024-02-06T22:16:44.669ZBegin loop 12
26092024-02-06T22:16:44.669ZDownstairs to restart: 1
26102024-02-06T22:16:44.699Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916025173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231}
26112024-02-06T22:16:44.699Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.91634046Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231}
26122024-02-06T22:16:44.699ZCrucible runtime is spawned
26132024-02-06T22:16:44.699Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916396642Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231}
26142024-02-06T22:16:44.699Z{"msg":"Crucible 5b2269ae-63c3-4df1-82dd-d87929a12753 has session id: c1764021-3d6b-464c-b239-454c6dc69b02","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916426828Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26152024-02-06T22:16:44.699Z{"msg":"Upstairs opts: Upstairs UUID: 5b2269ae-63c3-4df1-82dd-d87929a12753, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916481591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26162024-02-06T22:16:44.699Z{"msg":"Crucible stats registered with UUID: 5b2269ae-63c3-4df1-82dd-d87929a12753","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916513627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26172024-02-06T22:16:44.700Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916551711Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231}
26182024-02-06T22:16:44.700Z{"msg":"5b2269ae-63c3-4df1-82dd-d87929a12753 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916577948Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26192024-02-06T22:16:44.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916610314Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26202024-02-06T22:16:44.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916641699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26212024-02-06T22:16:44.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916676045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26222024-02-06T22:16:44.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.91670645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26232024-02-06T22:16:44.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916744775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26242024-02-06T22:16:44.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916776051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26252024-02-06T22:16:44.700Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916825794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26262024-02-06T22:16:44.700Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916866589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26272024-02-06T22:16:44.700Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.916962066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26282024-02-06T22:16:44.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.917068182Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26292024-02-06T22:16:44.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.917098637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26302024-02-06T22:16:44.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.917141332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26312024-02-06T22:16:44.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.917778026Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26322024-02-06T22:16:44.700Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.91782051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26332024-02-06T22:16:44.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.917845827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26342024-02-06T22:16:44.700Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.917884971Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26352024-02-06T22:16:44.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.918378875Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26362024-02-06T22:16:44.700Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.918419019Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26372024-02-06T22:16:44.700Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.918436877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26382024-02-06T22:16:44.701Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.918480301Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26392024-02-06T22:16:44.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919062332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26402024-02-06T22:16:44.701Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919098787Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26412024-02-06T22:16:44.701Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919115185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26422024-02-06T22:16:44.701Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919129013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26432024-02-06T22:16:44.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.91915006Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26442024-02-06T22:16:44.701Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919190065Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26452024-02-06T22:16:44.701Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919231309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26462024-02-06T22:16:44.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919610898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26472024-02-06T22:16:44.701Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919647843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26482024-02-06T22:16:44.701Z{"msg":"[0]R flush_numbers[0..12]: [47, 45, 44, 30, 49, 49, 48, 47, 44, 45, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919663581Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26492024-02-06T22:16:44.701Z{"msg":"[0]R generation[0..12]: [22, 21, 20, 16, 23, 23, 22, 22, 20, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919678479Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26502024-02-06T22:16:44.701Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919694137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26512024-02-06T22:16:44.701Z{"msg":"[1]R flush_numbers[0..12]: [47, 45, 44, 30, 49, 49, 48, 47, 44, 45, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919708965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26522024-02-06T22:16:44.701Z{"msg":"[1]R generation[0..12]: [22, 21, 20, 16, 23, 23, 22, 22, 20, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919722983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26532024-02-06T22:16:44.701Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.91974537Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26542024-02-06T22:16:44.701Z{"msg":"[2]R flush_numbers[0..12]: [47, 45, 44, 30, 49, 49, 48, 47, 44, 45, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919760178Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26552024-02-06T22:16:44.701Z{"msg":"[2]R generation[0..12]: [22, 21, 20, 16, 23, 23, 22, 22, 20, 21, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919774016Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26562024-02-06T22:16:44.701Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919800523Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26572024-02-06T22:16:44.701Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919814941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26582024-02-06T22:16:44.701Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
26592024-02-06T22:16:44.701Z,"time":"2024-02-06T22:16:43.919828769Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26602024-02-06T22:16:44.702Z{"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919849046Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26612024-02-06T22:16:44.702Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919862744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26622024-02-06T22:16:44.702Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919876462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26632024-02-06T22:16:44.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.91989558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26642024-02-06T22:16:44.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919916187Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26652024-02-06T22:16:44.702Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919938814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26662024-02-06T22:16:44.702Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919952832Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26672024-02-06T22:16:44.702Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.91996545Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26682024-02-06T22:16:44.702Z{"msg":"5b2269ae-63c3-4df1-82dd-d87929a12753 is now active with session: c1764021-3d6b-464c-b239-454c6dc69b02","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.919990237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26692024-02-06T22:16:44.702Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.920002465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231,"session_id":"c1764021-3d6b-464c-b239-454c6dc69b02"}
26702024-02-06T22:16:44.702Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:43.920014983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231}
26712024-02-06T22:16:44.702ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26722024-02-06T22:16:44.702ZLoading write count information from file "/tmp/test_repair_verify.data"
26732024-02-06T22:16:44.702ZRun Repair workload
26742024-02-06T22:16:44.702Z01/30 Write block 96 len 20480 data: 3 3 4 4 4
26752024-02-06T22:16:44.702Z02/30 Read block 569 len 40960
26762024-02-06T22:16:44.702Z03/30 Read block 102 len 36864
26772024-02-06T22:16:44.702Z04/30 Write block 595 len 4096 data: 5
26782024-02-06T22:16:44.702Z05/30 Write block 46 len 32768 data: 2 3 4 5 5 7 7 7
26792024-02-06T22:16:44.702Z06/30 Write block 517 len 24576 data: 4 3 2 2 2 3
26802024-02-06T22:16:44.702Z07/30 Write block 459 len 12288 data: 2 2 2
26812024-02-06T22:16:44.708Z08/30 Flush
26822024-02-06T22:16:44.708Z09/30 Write block 567 len 24576 data: 3 3 3 3 3 3
26832024-02-06T22:16:44.708Z10/30 Flush
26842024-02-06T22:16:44.709Z11/30 Write block 155 len 40960 data: 4 3 3 3 3 3 3 2 3 3
26852024-02-06T22:16:44.709Z12/30 Write block 467 len 24576 data: 3 3 3 3 4 4
26862024-02-06T22:16:44.709Z13/30 Read block 514 len 40960
26872024-02-06T22:16:44.709Z14/30 Read block 156 len 24576
26882024-02-06T22:16:44.709Z15/30 Read block 31 len 32768
26892024-02-06T22:16:44.709Z16/30 Read block 106 len 28672
26902024-02-06T22:16:44.709Z17/30 Read block 223 len 28672
26912024-02-06T22:16:44.709Z18/30 Read block 395 len 40960
26922024-02-06T22:16:44.709Z19/30 Write block 9 len 8192 data: 3 3
26932024-02-06T22:16:44.709Z20/30 Write block 112 len 36864 data: 3 3 4 5 4 6 7 7 5
26942024-02-06T22:16:44.709Z21/30 Write block 135 len 24576 data: 2 2 3 3 3 3
26952024-02-06T22:16:44.709Z22/30 Write block 393 len 12288 data: 3 3 4
26962024-02-06T22:16:44.709Z23/30 Write block 47 len 32768 data: 4 5 6 6 8 8 8 7
26972024-02-06T22:16:44.709Z24/30 Read block 423 len 28672
26982024-02-06T22:16:44.713Z25/30 Read block 224 len 20480
26992024-02-06T22:16:44.713Z26/30 Flush
27002024-02-06T22:16:44.728Z27/30 Write block 181 len 36864 data: 3 3 3 3 3 2 2 2 2
27012024-02-06T22:16:44.728Z28/30 Read block 331 len 28672
27022024-02-06T22:16:44.728Z29/30 Read block 580 len 16384
27032024-02-06T22:16:44.728Z30/30 Write block 86 len 24576 data: 2 2 2 2 2 2
27042024-02-06T22:16:44.728Z----------------------------------------------------------------
27052024-02-06T22:16:44.728Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
27062024-02-06T22:16:44.728ZStates: Active Active Active
27072024-02-06T22:16:44.728ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27082024-02-06T22:16:44.729Z 1 Acked 1000 Write 5 Done Sent Done false
27092024-02-06T22:16:44.729Z 2 Acked 1001 Read 10 Done Sent Done false
27102024-02-06T22:16:44.729Z 3 Acked 1002 Read 9 Done Sent Done false
27112024-02-06T22:16:44.729Z 4 Acked 1003 Write 1 Done Sent Done false
27122024-02-06T22:16:44.729Z 5 Acked 1004 Write 8 Done Sent Done false
27132024-02-06T22:16:44.729Z 6 Acked 1005 Write 6 Done Sent Done false
27142024-02-06T22:16:44.729Z 7 Acked 1006 Write 3 Done Sent Done false
27152024-02-06T22:16:44.729Z 8 Acked 1007 Flush 0 Done Sent Done false
27162024-02-06T22:16:44.729Z 9 Acked 1008 Write 6 Done Sent Done false
27172024-02-06T22:16:44.729Z 10 Acked 1009 Flush 0 Done Sent Done false
27182024-02-06T22:16:44.729Z 11 Acked 1010 Write 10 Done Sent Done false
27192024-02-06T22:16:44.729Z 12 Acked 1011 Write 6 Done Sent Done false
27202024-02-06T22:16:44.729Z 13 Acked 1012 Read 10 Done Sent Done false
27212024-02-06T22:16:44.729Z 14 Acked 1013 Read 6 Done Sent Done false
27222024-02-06T22:16:44.729Z 15 Acked 1014 Read 8 Done Sent Done false
27232024-02-06T22:16:44.729Z 16 Acked 1015 Read 7 Done Sent Done false
27242024-02-06T22:16:44.729Z 17 Acked 1016 Read 7 Done Sent Done false
27252024-02-06T22:16:44.729Z 18 Acked 1017 Read 10 Done Sent Done false
27262024-02-06T22:16:44.729Z 19 Acked 1018 Write 2 Done Sent Done false
27272024-02-06T22:16:44.729Z 20 Acked 1019 Write 9 Done Sent Done false
27282024-02-06T22:16:44.729Z 21 Acked 1020 Write 6 Done Sent Done false
27292024-02-06T22:16:44.729Z 22 Acked 1021 Write 3 Done Sent Done false
27302024-02-06T22:16:44.729Z 23 Acked 1022 Write 8 Done Sent Done false
27312024-02-06T22:16:44.729Z 24 Acked 1023 Read 7 Done Sent Done false
27322024-02-06T22:16:44.730Z 25 Acked 1024 Read 5 Done Sent Done false
27332024-02-06T22:16:44.730Z 26 Acked 1025 Flush 0 Done Sent Done false
27342024-02-06T22:16:44.730Z 27 Acked 1026 Write 9 Done Sent Done false
27352024-02-06T22:16:44.730Z 28 Acked 1027 Read 7 Done Sent Done false
27362024-02-06T22:16:44.730Z 29 Acked 1028 Read 4 Done Sent Done false
27372024-02-06T22:16:44.730Z 30 Acked 1029 Write 6 Sent Sent Sent false
27382024-02-06T22:16:44.730Z STATES DS:0 DS:1 DS:2 TOTAL
27392024-02-06T22:16:44.730Z New 0 0 0 0
27402024-02-06T22:16:44.730Z Sent 1 30 1 32
27412024-02-06T22:16:44.730Z Done 29 0 29 58
27422024-02-06T22:16:44.730Z Skipped 0 0 0 0
27432024-02-06T22:16:44.730Z Error 0 0 0 0
27442024-02-06T22:16:44.730ZLast Flush: 1025 0 1025
27452024-02-06T22:16:44.730ZDownstairs last five completed:
27462024-02-06T22:16:44.730ZUpstairs last five completed: 30 29 28 27 26
27472024-02-06T22:16:44.730Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:43.954486826Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1231}
27482024-02-06T22:16:44.730ZWrote out file "/tmp/test_repair_verify.data"
27492024-02-06T22:16:44.730Z
27502024-02-06T22:16:44.738ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27512024-02-06T22:16:44.745ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27522024-02-06T22:16:44.745Z 0 000-019 24 22 24 52 47 52 F F F <---
27532024-02-06T22:16:44.745Z 1 020-039 21 21 21 45 45 45 F F F
27542024-02-06T22:16:44.745Z 2 040-059 24 20 24 52 44 52 F F F <---
27552024-02-06T22:16:44.745Z 3 060-079 16 16 16 30 30 30 F F F
27562024-02-06T22:16:44.745Z 4 080-099 24 23 24 50 49 50 T F T <---
27572024-02-06T22:16:44.745Z 5 100-119 24 23 24 52 49 52 F F F <---
27582024-02-06T22:16:44.745Z 6 120-139 24 22 24 52 48 52 F F F <---
27592024-02-06T22:16:44.745Z 7 140-159 24 22 24 52 47 52 F F F <---
27602024-02-06T22:16:44.745Z 8 160-179 24 20 24 52 44 52 F F F <---
27612024-02-06T22:16:44.745Z 9 180-199 21 21 21 45 45 45 T F T <---
27622024-02-06T22:16:44.745Z 10 200-219 22 22 22 47 47 47 F F F
27632024-02-06T22:16:44.745Z 11 220-239 22 22 22 47 47 47 F F F
27642024-02-06T22:16:44.745Z 12 240-259 20 20 20 42 42 42 F F F
27652024-02-06T22:16:44.745Z 13 260-279 10 10 10 19 19 19 F F F
27662024-02-06T22:16:44.745Z 14 280-299 11 11 11 23 23 23 F F F
27672024-02-06T22:16:44.745Z 15 300-319 20 20 20 44 44 44 F F F
27682024-02-06T22:16:44.746Z 16 320-339 20 20 20 42 42 42 F F F
27692024-02-06T22:16:44.746Z 17 340-359 23 23 23 49 49 49 F F F
27702024-02-06T22:16:44.746Z 18 360-379 23 23 23 49 49 49 F F F
27712024-02-06T22:16:44.746Z 19 380-399 24 23 24 52 49 52 F F F <---
27722024-02-06T22:16:44.746Z 20 400-419 23 23 23 49 49 49 F F F
27732024-02-06T22:16:44.746Z 21 420-439 23 23 23 49 49 49 F F F
27742024-02-06T22:16:44.746Z 22 440-459 24 19 24 50 41 50 F F F <---
27752024-02-06T22:16:44.746Z 23 460-479 24 23 24 52 49 52 F F F <---
27762024-02-06T22:16:44.746Z 24 480-499 23 23 23 49 49 49 F F F
27772024-02-06T22:16:44.746Z 25 500-519 24 23 24 50 49 50 F F F <---
27782024-02-06T22:16:44.746Z 26 520-539 24 18 24 50 38 50 F F F <---
27792024-02-06T22:16:44.746Z 27 540-559 21 21 21 45 45 45 F F F
27802024-02-06T22:16:44.746Z 28 560-579 24 18 24 51 38 51 F F F <---
27812024-02-06T22:16:44.746Z 29 580-599 24 22 24 50 48 50 F F F <---
27822024-02-06T22:16:44.746ZMax gen: 24, Max flush: 52
27832024-02-06T22:16:44.746ZError: Difference in extent metadata found!
27842024-02-06T22:16:44.756ZA Difference in extent metadata is expected here
27852024-02-06T22:16:44.756ZOn loop 12
27862024-02-06T22:16:44.756Z
27872024-02-06T22:16:44.756ZVerifying data now
27882024-02-06T22:16:49.451ZNo change after verify
27892024-02-06T22:16:49.451ZLoop: 12 Downstairs dump after verify (and repair):
27902024-02-06T22:16:49.466ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27912024-02-06T22:16:49.466Z 0 000-019 24 24 24 52 52 52 F F F
27922024-02-06T22:16:49.466Z 1 020-039 21 21 21 45 45 45 F F F
27932024-02-06T22:16:49.467Z 2 040-059 24 24 24 52 52 52 F F F
27942024-02-06T22:16:49.467Z 3 060-079 16 16 16 30 30 30 F F F
27952024-02-06T22:16:49.467Z 4 080-099 25 25 25 53 53 53 F F F
27962024-02-06T22:16:49.467Z 5 100-119 24 24 24 52 52 52 F F F
27972024-02-06T22:16:49.467Z 6 120-139 24 24 24 52 52 52 F F F
27982024-02-06T22:16:49.467Z 7 140-159 24 24 24 52 52 52 F F F
27992024-02-06T22:16:49.467Z 8 160-179 24 24 24 52 52 52 F F F
28002024-02-06T22:16:49.471Z 9 180-199 25 25 25 53 53 53 F F F
28012024-02-06T22:16:49.471Z 10 200-219 22 22 22 47 47 47 F F F
28022024-02-06T22:16:49.471Z 11 220-239 22 22 22 47 47 47 F F F
28032024-02-06T22:16:49.471Z 12 240-259 20 20 20 42 42 42 F F F
28042024-02-06T22:16:49.471Z 13 260-279 10 10 10 19 19 19 F F F
28052024-02-06T22:16:49.471Z 14 280-299 11 11 11 23 23 23 F F F
28062024-02-06T22:16:49.471Z 15 300-319 20 20 20 44 44 44 F F F
28072024-02-06T22:16:49.471Z 16 320-339 20 20 20 42 42 42 F F F
28082024-02-06T22:16:49.471Z 17 340-359 23 23 23 49 49 49 F F F
28092024-02-06T22:16:49.471Z 18 360-379 23 23 23 49 49 49 F F F
28102024-02-06T22:16:49.472Z 19 380-399 24 24 24 52 52 52 F F F
28112024-02-06T22:16:49.472Z 20 400-419 23 23 23 49 49 49 F F F
28122024-02-06T22:16:49.472Z 21 420-439 23 23 23 49 49 49 F F F
28132024-02-06T22:16:49.472Z 22 440-459 24 24 24 50 50 50 F F F
28142024-02-06T22:16:49.472Z 23 460-479 24 24 24 52 52 52 F F F
28152024-02-06T22:16:49.472Z 24 480-499 23 23 23 49 49 49 F F F
28162024-02-06T22:16:49.472Z 25 500-519 24 24 24 50 50 50 F F F
28172024-02-06T22:16:49.472Z 26 520-539 24 24 24 50 50 50 F F F
28182024-02-06T22:16:49.472Z 27 540-559 21 21 21 45 45 45 F F F
28192024-02-06T22:16:49.472Z 28 560-579 24 24 24 51 51 51 F F F
28202024-02-06T22:16:49.472Z 29 580-599 24 24 24 50 50 50 F F F
28212024-02-06T22:16:49.472ZMax gen: 25, Max flush: 53
28222024-02-06T22:16:49.473Z
28232024-02-06T22:16:49.473ZBegin loop 13
28242024-02-06T22:16:49.473ZDownstairs to restart: 1
28252024-02-06T22:16:49.489Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.725677489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239}
28262024-02-06T22:16:49.495Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
28272024-02-06T22:16:49.495Z,"v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.725978818Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239}
28282024-02-06T22:16:49.495Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726016413Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239}
28292024-02-06T22:16:49.495Z{"msg":"Crucible 10203dfc-268c-45ed-ad43-dfe7918dd7c3 has session id: 3796341b-2cf4-4dcb-91e2-fad69d8cbe91","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.72604022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28302024-02-06T22:16:49.495Z{"msg":"Upstairs opts: Upstairs UUID: 10203dfc-268c-45ed-ad43-dfe7918dd7c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726085054Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28312024-02-06T22:16:49.495Z{"msg":"Crucible stats registered with UUID: 10203dfc-268c-45ed-ad43-dfe7918dd7c3","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726121049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28322024-02-06T22:16:49.495Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726145446Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239}
28332024-02-06T22:16:49.495Z{"msg":"10203dfc-268c-45ed-ad43-dfe7918dd7c3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726172032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28342024-02-06T22:16:49.495Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726194589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28352024-02-06T22:16:49.495Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726220975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28362024-02-06T22:16:49.495Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726253841Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28372024-02-06T22:16:49.495Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726325001Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28382024-02-06T22:16:49.495Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726373764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28392024-02-06T22:16:49.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726401931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28402024-02-06T22:16:49.496Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726425438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28412024-02-06T22:16:49.496Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726468362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28422024-02-06T22:16:49.496Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726504337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28432024-02-06T22:16:49.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726528664Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28442024-02-06T22:16:49.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726570078Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28452024-02-06T22:16:49.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.726595305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28462024-02-06T22:16:49.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.727187925Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28472024-02-06T22:16:49.496Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.727234918Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28482024-02-06T22:16:49.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.727270743Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28492024-02-06T22:16:49.496Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.72729312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28502024-02-06T22:16:49.496Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.72781685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28512024-02-06T22:16:49.496Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.727942143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28522024-02-06T22:16:49.496Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.728325051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28532024-02-06T22:16:49.496Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.728361936Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28542024-02-06T22:16:49.496Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.72840777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28552024-02-06T22:16:49.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.728498097Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28562024-02-06T22:16:49.496Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.728527544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28572024-02-06T22:16:49.496Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.728545311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28582024-02-06T22:16:49.496Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.728583496Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28592024-02-06T22:16:49.496Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.728602083Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28602024-02-06T22:16:49.496Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729038584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28612024-02-06T22:16:49.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729462997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28622024-02-06T22:16:49.501Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.72951589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28632024-02-06T22:16:49.501Z{"msg":"[0]R flush_numbers[0..12]: [52, 45, 52, 30, 53, 52, 52, 52, 52, 53, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729553955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28642024-02-06T22:16:49.501Z{"msg":"[0]R generation[0..12]: [24, 21, 24, 16, 25, 24, 24, 24, 24, 25, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729594029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28652024-02-06T22:16:49.501Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729612647Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28662024-02-06T22:16:49.501Z{"msg":"[1]R flush_numbers[0..12]: [52, 45, 52, 30, 53, 52, 52, 52, 52, 53, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729635104Z","hostname":"Wait for a query_work_queue command to finish before sending IO
28672024-02-06T22:16:49.501Zip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28682024-02-06T22:16:49.501Z{"msg":"[1]R generation[0..12]: [24, 21, 24, 16, 25, 24, 24, 24, 24, 25, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729652872Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28692024-02-06T22:16:49.502Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729681118Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28702024-02-06T22:16:49.502Z{"msg":"[2]R flush_numbers[0..12]: [52, 45, 52, 30, 53, 52, 52, 52, 52, 53, 47, 47]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729706464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28712024-02-06T22:16:49.502Z{"msg":"[2]R generation[0..12]: [24, 21, 24, 16, 25, 24, 24, 24, 24, 25, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729721652Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28722024-02-06T22:16:49.502Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729744969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28732024-02-06T22:16:49.502Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729759927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28742024-02-06T22:16:49.502Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729773735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28752024-02-06T22:16:49.502Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729794662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28762024-02-06T22:16:49.502Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729809111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28772024-02-06T22:16:49.502Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729830608Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28782024-02-06T22:16:49.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729842886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28792024-02-06T22:16:49.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729856284Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28802024-02-06T22:16:49.502Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729876911Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28812024-02-06T22:16:49.502Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729891379Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28822024-02-06T22:16:49.502Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729911137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28832024-02-06T22:16:49.502Z{"msg":"10203dfc-268c-45ed-ad43-dfe7918dd7c3 is now active with session: 3796341b-2cf4-4dcb-91e2-fad69d8cbe91","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729924965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28842024-02-06T22:16:49.502Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729938343Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239,"session_id":"3796341b-2cf4-4dcb-91e2-fad69d8cbe91"}
28852024-02-06T22:16:49.502Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:48.729950501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239}
28862024-02-06T22:16:49.502ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28872024-02-06T22:16:49.502ZLoading write count information from file "/tmp/test_repair_verify.data"
28882024-02-06T22:16:49.502ZRun Repair workload
28892024-02-06T22:16:49.502Z01/30 Write block 482 len 16384 data: 2 2 2 3
28902024-02-06T22:16:49.502Z02/30 Read block 390 len 12288
28912024-02-06T22:16:49.502Z03/30 Flush
28922024-02-06T22:16:49.502Z04/30 Write block 354 len 16384 data: 4 4 4 6
28932024-02-06T22:16:49.503Z05/30 Write block 506 len 12288 data: 4 4 4
28942024-02-06T22:16:49.503Z06/30 Read block 290 len 32768
28952024-02-06T22:16:49.506Z07/30 Read block 331 len 16384
28962024-02-06T22:16:49.506Z08/30 Read block 556 len 4096
28972024-02-06T22:16:49.506Z09/30 Read block 549 len 32768
28982024-02-06T22:16:49.506Z10/30 Read block 584 len 4096
28992024-02-06T22:16:49.506Z11/30 Flush
29002024-02-06T22:16:49.506Z12/30 Write block 541 len 40960 data: 3 3 3 3 3 3 3 3 4 3
29012024-02-06T22:16:49.509Z13/30 Write block 151 len 36864 data: 4 4 6 4 5 4 4 4 4
29022024-02-06T22:16:49.509Z14/30 Read block 196 len 40960
29032024-02-06T22:16:49.509Z15/30 Flush
29042024-02-06T22:16:49.513Z16/30 Write block 225 len 32768 data: 4 4 4 4 3 2 2 2
29052024-02-06T22:16:49.513Z17/30 Write block 432 len 36864 data: 3 3 4 4 4 4 4 4 4
29062024-02-06T22:16:49.513Z18/30 Read block 341 len 24576
29072024-02-06T22:16:49.517Z19/30 Write block 179 len 40960 data: 3 3 4 4 4 4 4 3 3 3
29082024-02-06T22:16:49.517Z20/30 Read block 562 len 40960
29092024-02-06T22:16:49.517Z21/30 Write block 312 len 16384 data: 4 5 6 6
29102024-02-06T22:16:49.520Z22/30 Read block 589 len 40960
29112024-02-06T22:16:49.520Z23/30 Flush
29122024-02-06T22:16:49.523Z24/30 Flush
29132024-02-06T22:16:49.523Z25/30 Write block 279 len 24576 data: 2 2 2 2 2 2
29142024-02-06T22:16:49.523Z26/30 Read block 215 len 36864
29152024-02-06T22:16:49.523Z27/30 Read block 346 len 8192
29162024-02-06T22:16:49.526Z28/30 Read block 12 len 24576
29172024-02-06T22:16:49.526Z29/30 Read block 430 len 36864
29182024-02-06T22:16:49.526Z30/30 Write block 102 len 28672 data: 3 3 3 3 2 2 3
29192024-02-06T22:16:49.529Z----------------------------------------------------------------
29202024-02-06T22:16:49.529Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29212024-02-06T22:16:49.529ZStates: Active Active Active
29222024-02-06T22:16:49.529ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29232024-02-06T22:16:49.529Z 1 Acked 1000 Write 4 Done Sent Done false
29242024-02-06T22:16:49.530Z 2 Acked 1001 Read 3 Done Sent Done false
29252024-02-06T22:16:49.530Z 3 Acked 1002 Flush 0 Done Sent Done false
29262024-02-06T22:16:49.530Z 4 Acked 1003 Write 4 Done Sent Done false
29272024-02-06T22:16:49.530Z 5 Acked 1004 Write 3 Done Sent Done false
29282024-02-06T22:16:49.530Z 6 Acked 1005 Read 8 Done Sent Done false
29292024-02-06T22:16:49.530Z 7 Acked 1006 Read 4 Done Sent Done false
29302024-02-06T22:16:49.530Z 8 Acked 1007 Read 1 Done Sent Done false
29312024-02-06T22:16:49.530Z 9 Acked 1008 Read 8 Done Sent Done false
29322024-02-06T22:16:49.530Z 10 Acked 1009 Read 1 Done Sent Done false
29332024-02-06T22:16:49.530Z 11 Acked 1010 Flush 0 Done Sent Done false
29342024-02-06T22:16:49.530Z 12 Acked 1011 Write 10 Done Sent Done false
29352024-02-06T22:16:49.530Z 13 Acked 1012 Write 9 Done Sent Done false
29362024-02-06T22:16:49.530Z 14 Acked 1013 Read 10 Done Sent Done false
29372024-02-06T22:16:49.530Z 15 Acked 1014 Flush 0 Done Sent Done false
29382024-02-06T22:16:49.530Z 16 Acked 1015 Write 8 Done Sent Done false
29392024-02-06T22:16:49.530Z 17 Acked 1016 Write 9 Done Sent Done false
29402024-02-06T22:16:49.530Z 18 Acked 1017 Read 6 Done Sent Done false
29412024-02-06T22:16:49.530Z 19 Acked 1018 Write 10 Done Sent Done false
29422024-02-06T22:16:49.530Z 20 Acked 1019 Read 10 Done Sent Done false
29432024-02-06T22:16:49.530Z 21 Acked 1020 Write 4 Done Sent Done false
29442024-02-06T22:16:49.530Z 22 Acked 1021 Read 10 Done Sent Done false
29452024-02-06T22:16:49.530Z 23 Acked 1022 Flush 0 Done Sent Done false
29462024-02-06T22:16:49.530Z 24 Acked 1023 Flush 0 Done Sent Done false
29472024-02-06T22:16:49.530Z 25 Acked 1024 Write 6 Done Sent Done false
29482024-02-06T22:16:49.531Z 26 Acked 1025 Read 9 Done Sent Done false
29492024-02-06T22:16:49.531Z 27 Acked 1026 Read 2 Done Sent Done false
29502024-02-06T22:16:49.531Z 28 Acked 1027 Read 6 Done Sent Done false
29512024-02-06T22:16:49.531Z 29 Acked 1028 Read 9 Done Sent Done false
29522024-02-06T22:16:49.531Z 30 Acked 1029 Write 7 Sent Sent Sent false
29532024-02-06T22:16:49.531Z STATES DS:0 DS:1 DS:2 TOTAL
29542024-02-06T22:16:49.531Z New 0 0 0 0
29552024-02-06T22:16:49.531Z Sent 1 30 1 32
29562024-02-06T22:16:49.531Z Done 29 0 29 58
29572024-02-06T22:16:49.531Z Skipped 0 0 0 0
29582024-02-06T22:16:49.531Z Error 0 0 0 0
29592024-02-06T22:16:49.531ZLast Flush: 1023 0 1023
29602024-02-06T22:16:49.531ZDownstairs last five completed:
29612024-02-06T22:16:49.531ZUpstairs last five completed: 30 29 28 27 26
29622024-02-06T22:16:49.531Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:48.76259757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1239}
29632024-02-06T22:16:49.531ZWrote out file "/tmp/test_repair_verify.data"
29642024-02-06T22:16:49.534Z
29652024-02-06T22:16:49.535ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29662024-02-06T22:16:49.549ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29672024-02-06T22:16:49.549Z 0 000-019 24 24 24 52 52 52 F F F
29682024-02-06T22:16:49.550Z 1 020-039 21 21 21 45 45 45 F F F
29692024-02-06T22:16:49.550Z 2 040-059 24 24 24 52 52 52 F F F
29702024-02-06T22:16:49.550Z 3 060-079 16 16 16 30 30 30 F F F
29712024-02-06T22:16:49.550Z 4 080-099 25 25 25 53 53 53 F F F
29722024-02-06T22:16:49.550Z 5 100-119 24 24 24 52 52 52 T F T <---
29732024-02-06T22:16:49.550Z 6 120-139 24 24 24 52 52 52 F F F
29742024-02-06T22:16:49.550Z 7 140-159 26 24 26 56 52 56 F F F <---
29752024-02-06T22:16:49.550Z 8 160-179 26 24 26 57 52 57 F F F <---
29762024-02-06T22:16:49.550Z 9 180-199 26 25 26 57 53 57 F F F <---
29772024-02-06T22:16:49.550Z 10 200-219 22 22 22 47 47 47 F F F
29782024-02-06T22:16:49.550Z 11 220-239 26 22 26 57 47 57 F F F <---
29792024-02-06T22:16:49.553Z 12 240-259 20 20 20 42 42 42 F F F
29802024-02-06T22:16:49.553Z 13 260-279 10 10 10 19 19 19 T F T <---
29812024-02-06T22:16:49.554Z 14 280-299 11 11 11 23 23 23 T F T <---
29822024-02-06T22:16:49.554Z 15 300-319 26 20 26 57 44 57 F F F <---
29832024-02-06T22:16:49.554Z 16 320-339 20 20 20 42 42 42 F F F
29842024-02-06T22:16:49.554Z 17 340-359 26 23 26 55 49 55 F F F <---
29852024-02-06T22:16:49.554Z 18 360-379 23 23 23 49 49 49 F F F
29862024-02-06T22:16:49.554Z 19 380-399 24 24 24 52 52 52 F F F
29872024-02-06T22:16:49.554Z 20 400-419 23 23 23 49 49 49 F F F
29882024-02-06T22:16:49.554Z 21 420-439 26 23 26 57 49 57 F F F <---
29892024-02-06T22:16:49.554Z 22 440-459 26 24 26 57 50 57 F F F <---
29902024-02-06T22:16:49.554Z 23 460-479 24 24 24 52 52 52 F F F
29912024-02-06T22:16:49.554Z 24 480-499 26 23 26 54 49 54 F F F <---
29922024-02-06T22:16:49.554Z 25 500-519 26 24 26 55 50 55 F F F <---
29932024-02-06T22:16:49.554Z 26 520-539 24 24 24 50 50 50 F F F
29942024-02-06T22:16:49.554Z 27 540-559 26 21 26 56 45 56 F F F <---
29952024-02-06T22:16:49.554Z 28 560-579 24 24 24 51 51 51 F F F
29962024-02-06T22:16:49.554Z 29 580-599 24 24 24 50 50 50 F F F
29972024-02-06T22:16:49.554ZMax gen: 26, Max flush: 57
29982024-02-06T22:16:49.554ZError: Difference in extent metadata found!
29992024-02-06T22:16:49.554ZA Difference in extent metadata is expected here
30002024-02-06T22:16:49.554ZOn loop 13
30012024-02-06T22:16:49.554Z
30022024-02-06T22:16:49.558ZVerifying data now
30032024-02-06T22:16:54.221ZNo change after verify
30042024-02-06T22:16:54.221ZLoop: 13 Downstairs dump after verify (and repair):
30052024-02-06T22:16:54.236ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30062024-02-06T22:16:54.236Z 0 000-019 24 24 24 52 52 52 F F F
30072024-02-06T22:16:54.236Z 1 020-039 21 21 21 45 45 45 F F F
30082024-02-06T22:16:54.236Z 2 040-059 24 24 24 52 52 52 F F F
30092024-02-06T22:16:54.236Z 3 060-079 16 16 16 30 30 30 F F F
30102024-02-06T22:16:54.236Z 4 080-099 25 25 25 53 53 53 F F F
30112024-02-06T22:16:54.236Z 5 100-119 27 27 27 58 58 58 F F F
30122024-02-06T22:16:54.237Z 6 120-139 24 24 24 52 52 52 F F F
30132024-02-06T22:16:54.237Z 7 140-159 26 26 26 56 56 56 F F F
30142024-02-06T22:16:54.237Z 8 160-179 26 26 26 57 57 57 F F F
30152024-02-06T22:16:54.237Z 9 180-199 26 26 26 57 57 57 F F F
30162024-02-06T22:16:54.237Z 10 200-219 22 22 22 47 47 47 F F F
30172024-02-06T22:16:54.237Z 11 220-239 26 26 26 57 57 57 F F F
30182024-02-06T22:16:54.237Z 12 240-259 20 20 20 42 42 42 F F F
30192024-02-06T22:16:54.237Z 13 260-279 27 27 27 58 58 58 F F F
30202024-02-06T22:16:54.237Z 14 280-299 27 27 27 58 58 58 F F F
30212024-02-06T22:16:54.237Z 15 300-319 26 26 26 57 57 57 F F F
30222024-02-06T22:16:54.240Z 16 320-339 20 20 20 42 42 42 F F F
30232024-02-06T22:16:54.240Z 17 340-359 26 26 26 55 55 55 F F F
30242024-02-06T22:16:54.240Z 18 360-379 23 23 23 49 49 49 F F F
30252024-02-06T22:16:54.240Z 19 380-399 24 24 24 52 52 52 F F F
30262024-02-06T22:16:54.240Z 20 400-419 23 23 23 49 49 49 F F F
30272024-02-06T22:16:54.240Z 21 420-439 26 26 26 57 57 57 F F F
30282024-02-06T22:16:54.240Z 22 440-459 26 26 26 57 57 57 F F F
30292024-02-06T22:16:54.240Z 23 460-479 24 24 24 52 52 52 F F F
30302024-02-06T22:16:54.241Z 24 480-499 26 26 26 54 54 54 F F F
30312024-02-06T22:16:54.241Z 25 500-519 26 26 26 55 55 55 F F F
30322024-02-06T22:16:54.241Z 26 520-539 24 24 24 50 50 50 F F F
30332024-02-06T22:16:54.241Z 27 540-559 26 26 26 56 56 56 F F F
30342024-02-06T22:16:54.241Z 28 560-579 24 24 24 51 51 51 F F F
30352024-02-06T22:16:54.241Z 29 580-599 24 24 24 50 50 50 F F F
30362024-02-06T22:16:54.241ZMax gen: 27, Max flush: 58
30372024-02-06T22:16:54.241Z
30382024-02-06T22:16:54.241ZBegin loop 14
30392024-02-06T22:16:54.241ZDownstairs to restart: 2
30402024-02-06T22:16:54.257Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.491984443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247}
30412024-02-06T22:16:54.257Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
30422024-02-06T22:16:54.257Z,"time":"2024-02-06T22:16:53.492297541Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247}
30432024-02-06T22:16:54.262Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492334505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247}
30442024-02-06T22:16:54.262Z{"msg":"Crucible 63d07d1c-2b76-4ab1-984c-82b7a9ab9ca4 has session id: 6b4b906a-b188-40c0-b1fd-ac1585b216be","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492389528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30452024-02-06T22:16:54.263Z{"msg":"Upstairs opts: Upstairs UUID: 63d07d1c-2b76-4ab1-984c-82b7a9ab9ca4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492453509Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30462024-02-06T22:16:54.263Z{"msg":"Crucible stats registered with UUID: 63d07d1c-2b76-4ab1-984c-82b7a9ab9ca4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492488295Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30472024-02-06T22:16:54.263Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492510392Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247}
30482024-02-06T22:16:54.263Z{"msg":"63d07d1c-2b76-4ab1-984c-82b7a9ab9ca4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492550026Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30492024-02-06T22:16:54.263Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492581012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30502024-02-06T22:16:54.263Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492625816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30512024-02-06T22:16:54.263Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.4926676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30522024-02-06T22:16:54.263Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492721043Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30532024-02-06T22:16:54.263Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492754909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30542024-02-06T22:16:54.263Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492800553Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30552024-02-06T22:16:54.263Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492844887Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30562024-02-06T22:16:54.263Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492884181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30572024-02-06T22:16:54.263Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492914207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30582024-02-06T22:16:54.263Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492938854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30592024-02-06T22:16:54.263Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.492987757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30602024-02-06T22:16:54.263Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.493009954Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30612024-02-06T22:16:54.263Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.493561979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30622024-02-06T22:16:54.263Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.493603404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30632024-02-06T22:16:54.263Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.49363241Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30642024-02-06T22:16:54.263Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.493719268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30652024-02-06T22:16:54.263Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494126563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30662024-02-06T22:16:54.263Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494250877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30672024-02-06T22:16:54.263Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494422833Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30682024-02-06T22:16:54.264Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494491894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30692024-02-06T22:16:54.264Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494639194Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30702024-02-06T22:16:54.264Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494689377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30712024-02-06T22:16:54.264Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494704795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30722024-02-06T22:16:54.264Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494718093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30732024-02-06T22:16:54.264Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.494731792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30742024-02-06T22:16:54.264Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.49474542Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30752024-02-06T22:16:54.264Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495000075Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30762024-02-06T22:16:54.264Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495433697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30772024-02-06T22:16:54.264Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495466382Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30782024-02-06T22:16:54.264Z{"msg":"[0]R flush_numbers[0..12]: [52, 45, 52, 30, 53, 58, 52, 56, 57, 57, 47, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.49548163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30792024-02-06T22:16:54.264Z{"msg":"[0]R generation[0..12]: [24, 21, 24, 16, 25, 27, 24, 26, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495495928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30802024-02-06T22:16:54.264Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495518045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30812024-02-06T22:16:54.264Z{"msg":"[1]R flush_numbers[0..12]: [52, 45, 52, 30, 53, 58, 52, 56, 57, 57, 47, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495532443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30822024-02-06T22:16:54.264Z{"msg":"[1]R generation[0..12]: [24, 21, 24, 16, 25, 27, 24, 26, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495546431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30832024-02-06T22:16:54.264Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495568159Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30842024-02-06T22:16:54.264Z{"msg":"[2]R flush_numbers[0..12]: [52, 45, 52, 30, 53, 58, 52, 56, 57, 57, 47, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495582407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30852024-02-06T22:16:54.264Z{"msg":"[2]R generation[0..12]: [24, 21, 24, 16, 25, 27, 24, 26, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495603524Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30862024-02-06T22:16:54.264Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495618172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30872024-02-06T22:16:54.264Z{"msg":"Max found gen is 28","v":0,"Wait for a query_work_queue command to finish before sending IO
30882024-02-06T22:16:54.264Zname":"crucible","level":30,"time":"2024-02-06T22:16:53.495642778Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30892024-02-06T22:16:54.264Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495657127Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30902024-02-06T22:16:54.264Z{"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495670245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30912024-02-06T22:16:54.264Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495683223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30922024-02-06T22:16:54.265Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.49570403Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30932024-02-06T22:16:54.265Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495716528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30942024-02-06T22:16:54.270Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495729877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30952024-02-06T22:16:54.270Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495755763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30962024-02-06T22:16:54.270Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495769431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30972024-02-06T22:16:54.271Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.49578188Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30982024-02-06T22:16:54.271Z{"msg":"63d07d1c-2b76-4ab1-984c-82b7a9ab9ca4 is now active with session: 6b4b906a-b188-40c0-b1fd-ac1585b216be","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495801757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
30992024-02-06T22:16:54.271Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495813995Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247,"session_id":"6b4b906a-b188-40c0-b1fd-ac1585b216be"}
31002024-02-06T22:16:54.271Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:53.495826274Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247}
31012024-02-06T22:16:54.271ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
31022024-02-06T22:16:54.271ZLoading write count information from file "/tmp/test_repair_verify.data"
31032024-02-06T22:16:54.271ZRun Repair workload
31042024-02-06T22:16:54.272Z01/30 Write block 567 len 20480 data: 4 4 4 4 4
31052024-02-06T22:16:54.272Z02/30 Write block 406 len 8192 data: 2 2
31062024-02-06T22:16:54.272Z03/30 Read block 439 len 8192
31072024-02-06T22:16:54.272Z04/30 Read block 492 len 4096
31082024-02-06T22:16:54.272Z05/30 Read block 76 len 4096
31092024-02-06T22:16:54.273Z06/30 Write block 415 len 20480 data: 3 3 5 5 4
31102024-02-06T22:16:54.273Z07/30 Read block 372 len 8192
31112024-02-06T22:16:54.273Z08/30 Read block 332 len 20480
31122024-02-06T22:16:54.273Z09/30 Write block 276 len 32768 data: 2 2 2 3 3 3 3 3
31132024-02-06T22:16:54.287Z10/30 Read block 130 len 8192
31142024-02-06T22:16:54.287Z11/30 Write block 403 len 8192 data: 2 2
31152024-02-06T22:16:54.287Z12/30 Flush
31162024-02-06T22:16:54.287Z13/30 Write block 139 len 4096 data: 4
31172024-02-06T22:16:54.287Z14/30 Write block 426 len 20480 data: 3 3 4 4 5
31182024-02-06T22:16:54.287Z15/30 Read block 347 len 20480
31192024-02-06T22:16:54.287Z16/30 Write block 19 len 32768 data: 3 2 2 3 3 3 4 5
31202024-02-06T22:16:54.287Z17/30 Read block 549 len 8192
31212024-02-06T22:16:54.287Z18/30 Write block 572 len 36864 data: 4 3 3 3 3 3 3 4 5
31222024-02-06T22:16:54.287Z19/30 Read block 211 len 20480
31232024-02-06T22:16:54.287Z20/30 Write block 334 len 20480 data: 4 4 4 4 5
31242024-02-06T22:16:54.288Z21/30 Read block 563 len 12288
31252024-02-06T22:16:54.288Z22/30 Read block 225 len 16384
31262024-02-06T22:16:54.288Z23/30 Read block 267 len 40960
31272024-02-06T22:16:54.297Z24/30 Read block 512 len 36864
31282024-02-06T22:16:54.297Z25/30 Read block 30 len 28672
31292024-02-06T22:16:54.297Z26/30 Read block 246 len 20480
31302024-02-06T22:16:54.297Z27/30 Read block 71 len 40960
31312024-02-06T22:16:54.297Z28/30 Write block 293 len 40960 data: 5 5 6 4 4 3 3 4 3 3
31322024-02-06T22:16:54.297Z29/30 Read block 172 len 20480
31332024-02-06T22:16:54.297Z30/30 Write block 284 len 16384 data: 3 2 2 2
31342024-02-06T22:16:54.297Z----------------------------------------------------------------
31352024-02-06T22:16:54.297Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31362024-02-06T22:16:54.297ZStates: Active Active Active
31372024-02-06T22:16:54.297ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31382024-02-06T22:16:54.297Z 1 Acked 1000 Write 5 Done Done Done false
31392024-02-06T22:16:54.297Z 2 Acked 1001 Write 2 Done Done Done false
31402024-02-06T22:16:54.297Z 3 Acked 1002 Read 2 Done Done Done false
31412024-02-06T22:16:54.297Z 4 Acked 1003 Read 1 Done Done Done false
31422024-02-06T22:16:54.298Z 5 Acked 1004 Read 1 Done Done Sent false
31432024-02-06T22:16:54.298Z 6 Acked 1005 Write 5 Done Done Sent false
31442024-02-06T22:16:54.298Z 7 Acked 1006 Read 2 Done Done Sent false
31452024-02-06T22:16:54.298Z 8 Acked 1007 Read 5 Done Done Sent false
31462024-02-06T22:16:54.298Z 9 Acked 1008 Write 8 Done Done Sent false
31472024-02-06T22:16:54.298Z 10 Acked 1009 Read 2 Done Done Sent false
31482024-02-06T22:16:54.298Z 11 Acked 1010 Write 2 Done Done Sent false
31492024-02-06T22:16:54.298Z 12 Acked 1011 Flush 0 Done Done Sent false
31502024-02-06T22:16:54.298Z 13 Acked 1012 Write 1 Done Done Sent false
31512024-02-06T22:16:54.298Z 14 Acked 1013 Write 5 Done Done Sent false
31522024-02-06T22:16:54.298Z 15 Acked 1014 Read 5 Done Done Sent false
31532024-02-06T22:16:54.298Z 16 Acked 1015 Write 8 Done Done Sent false
31542024-02-06T22:16:54.298Z 17 Acked 1016 Read 2 Done Done Sent false
31552024-02-06T22:16:54.298Z 18 Acked 1017 Write 9 Done Done Sent false
31562024-02-06T22:16:54.298Z 19 Acked 1018 Read 5 Done Done Sent false
31572024-02-06T22:16:54.298Z 20 Acked 1019 Write 5 Done Done Sent false
31582024-02-06T22:16:54.298Z 21 Acked 1020 Read 3 Done Done Sent false
31592024-02-06T22:16:54.298Z 22 Acked 1021 Read 4 Done Done Sent false
31602024-02-06T22:16:54.298Z 23 Acked 1022 Read 10 Done Done Sent false
31612024-02-06T22:16:54.298Z 24 Acked 1023 Read 9 Done Done Sent false
31622024-02-06T22:16:54.298Z 25 Acked 1024 Read 7 Done Done Sent false
31632024-02-06T22:16:54.298Z 26 Acked 1025 Read 5 Done Done Sent false
31642024-02-06T22:16:54.298Z 27 Acked 1026 Read 10 Done Done Sent false
31652024-02-06T22:16:54.298Z 28 Acked 1027 Write 10 Done Done Sent false
31662024-02-06T22:16:54.298Z 29 Acked 1028 Read 5 Done Done Sent false
31672024-02-06T22:16:54.299Z 30 Acked 1029 Write 4 Sent Sent Sent false
31682024-02-06T22:16:54.299Z STATES DS:0 DS:1 DS:2 TOTAL
31692024-02-06T22:16:54.299Z New 0 0 0 0
31702024-02-06T22:16:54.299Z Sent 1 1 26 28
31712024-02-06T22:16:54.299Z Done 29 29 4 62
31722024-02-06T22:16:54.299Z Skipped 0 0 0 0
31732024-02-06T22:16:54.299Z Error 0 0 0 0
31742024-02-06T22:16:54.299ZLast Flush: 1011 1011 0
31752024-02-06T22:16:54.299ZDownstairs last five completed:
31762024-02-06T22:16:54.299ZUpstairs last five completed: 30 29 28 27 26
31772024-02-06T22:16:54.299Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:53.524503529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1247}
31782024-02-06T22:16:54.299ZWrote out file "/tmp/test_repair_verify.data"
31792024-02-06T22:16:54.299Z
31802024-02-06T22:16:54.302ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31812024-02-06T22:16:54.314ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31822024-02-06T22:16:54.314Z 0 000-019 24 24 24 52 52 52 T T F <---
31832024-02-06T22:16:54.314Z 1 020-039 21 21 21 45 45 45 T T F <---
31842024-02-06T22:16:54.314Z 2 040-059 24 24 24 52 52 52 F F F
31852024-02-06T22:16:54.314Z 3 060-079 16 16 16 30 30 30 F F F
31862024-02-06T22:16:54.314Z 4 080-099 25 25 25 53 53 53 F F F
31872024-02-06T22:16:54.314Z 5 100-119 27 27 27 58 58 58 F F F
31882024-02-06T22:16:54.314Z 6 120-139 24 24 24 52 52 52 T T F <---
31892024-02-06T22:16:54.315Z 7 140-159 26 26 26 56 56 56 F F F
31902024-02-06T22:16:54.315Z 8 160-179 26 26 26 57 57 57 F F F
31912024-02-06T22:16:54.315Z 9 180-199 26 26 26 57 57 57 F F F
31922024-02-06T22:16:54.315Z 10 200-219 22 22 22 47 47 47 F F F
31932024-02-06T22:16:54.315Z 11 220-239 26 26 26 57 57 57 F F F
31942024-02-06T22:16:54.318Z 12 240-259 20 20 20 42 42 42 F F F
31952024-02-06T22:16:54.318Z 13 260-279 28 28 27 59 59 58 F F F <---
31962024-02-06T22:16:54.318Z 14 280-299 28 28 27 59 59 58 T T F <---
31972024-02-06T22:16:54.318Z 15 300-319 26 26 26 57 57 57 T T F <---
31982024-02-06T22:16:54.318Z 16 320-339 20 20 20 42 42 42 T T F <---
31992024-02-06T22:16:54.318Z 17 340-359 26 26 26 55 55 55 F F F
32002024-02-06T22:16:54.318Z 18 360-379 23 23 23 49 49 49 F F F
32012024-02-06T22:16:54.318Z 19 380-399 24 24 24 52 52 52 F F F
32022024-02-06T22:16:54.318Z 20 400-419 28 28 23 59 59 49 F F T <---
32032024-02-06T22:16:54.318Z 21 420-439 26 26 26 57 57 57 T T F <---
32042024-02-06T22:16:54.319Z 22 440-459 26 26 26 57 57 57 F F F
32052024-02-06T22:16:54.319Z 23 460-479 24 24 24 52 52 52 F F F
32062024-02-06T22:16:54.319Z 24 480-499 26 26 26 54 54 54 F F F
32072024-02-06T22:16:54.319Z 25 500-519 26 26 26 55 55 55 F F F
32082024-02-06T22:16:54.319Z 26 520-539 24 24 24 50 50 50 F F F
32092024-02-06T22:16:54.319Z 27 540-559 26 26 26 56 56 56 F F F
32102024-02-06T22:16:54.319Z 28 560-579 28 28 24 59 59 51 T T T <---
32112024-02-06T22:16:54.319Z 29 580-599 24 24 24 50 50 50 T T F <---
32122024-02-06T22:16:54.319ZMax gen: 28, Max flush: 59
32132024-02-06T22:16:54.319ZError: Difference in extent metadata found!
32142024-02-06T22:16:54.319ZA Difference in extent metadata is expected here
32152024-02-06T22:16:54.319ZOn loop 14
32162024-02-06T22:16:54.319Z
32172024-02-06T22:16:54.324ZVerifying data now
32182024-02-06T22:16:59.887ZNo change after verify
32192024-02-06T22:16:59.887ZLoop: 14 Downstairs dump after verify (and repair):
32202024-02-06T22:16:59.897ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32212024-02-06T22:16:59.897Z 0 000-019 29 29 29 60 60 60 F F F
32222024-02-06T22:16:59.897Z 1 020-039 29 29 29 60 60 60 F F F
32232024-02-06T22:16:59.897Z 2 040-059 24 24 24 52 52 52 F F F
32242024-02-06T22:16:59.897Z 3 060-079 16 16 16 30 30 30 F F F
32252024-02-06T22:16:59.897Z 4 080-099 25 25 25 53 53 53 F F F
32262024-02-06T22:16:59.897Z 5 100-119 27 27 27 58 58 58 F F F
32272024-02-06T22:16:59.897Z 6 120-139 29 29 29 60 60 60 F F F
32282024-02-06T22:16:59.897Z 7 140-159 26 26 26 56 56 56 F F F
32292024-02-06T22:16:59.897Z 8 160-179 26 26 26 57 57 57 F F F
32302024-02-06T22:16:59.897Z 9 180-199 26 26 26 57 57 57 F F F
32312024-02-06T22:16:59.900Z 10 200-219 22 22 22 47 47 47 F F F
32322024-02-06T22:16:59.900Z 11 220-239 26 26 26 57 57 57 F F F
32332024-02-06T22:16:59.900Z 12 240-259 20 20 20 42 42 42 F F F
32342024-02-06T22:16:59.901Z 13 260-279 28 28 28 59 59 59 F F F
32352024-02-06T22:16:59.901Z 14 280-299 29 29 29 60 60 60 F F F
32362024-02-06T22:16:59.901Z 15 300-319 29 29 29 60 60 60 F F F
32372024-02-06T22:16:59.901Z 16 320-339 29 29 29 60 60 60 F F F
32382024-02-06T22:16:59.901Z 17 340-359 26 26 26 55 55 55 F F F
32392024-02-06T22:16:59.901Z 18 360-379 23 23 23 49 49 49 F F F
32402024-02-06T22:16:59.901Z 19 380-399 24 24 24 52 52 52 F F F
32412024-02-06T22:16:59.901Z 20 400-419 28 28 28 59 59 59 F F F
32422024-02-06T22:16:59.901Z 21 420-439 29 29 29 60 60 60 F F F
32432024-02-06T22:16:59.901Z 22 440-459 26 26 26 57 57 57 F F F
32442024-02-06T22:16:59.901Z 23 460-479 24 24 24 52 52 52 F F F
32452024-02-06T22:16:59.901Z 24 480-499 26 26 26 54 54 54 F F F
32462024-02-06T22:16:59.901Z 25 500-519 26 26 26 55 55 55 F F F
32472024-02-06T22:16:59.901Z 26 520-539 24 24 24 50 50 50 F F F
32482024-02-06T22:16:59.901Z 27 540-559 26 26 26 56 56 56 F F F
32492024-02-06T22:16:59.901Z 28 560-579 29 29 29 60 60 60 F F F
32502024-02-06T22:16:59.901Z 29 580-599 29 29 29 60 60 60 F F F
32512024-02-06T22:16:59.901ZMax gen: 29, Max flush: 60
32522024-02-06T22:16:59.901Z
32532024-02-06T22:16:59.901ZBegin loop 15
32542024-02-06T22:16:59.901ZDownstairs to restart: 2
32552024-02-06T22:16:59.918Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.152536952Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255}
32562024-02-06T22:16:59.918Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.152789358Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255}
32572024-02-06T22:16:59.918Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.152817874Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255}
32582024-02-06T22:16:59.924Z{"msg":"Crucible ca0281f5-0365-4aad-9661-bd62eccadaea has session id: dbcce7b6-c24d-45de-aca5-27eee60f2d25","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.152852699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32592024-02-06T22:16:59.924Z{"msg":"Upstairs opts: Upstairs UUID: ca0281f5-0365-4aad-9661-bd62eccadaea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.152907602Z","hostnameCrucible runtime is spawned
32602024-02-06T22:16:59.924Z":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32612024-02-06T22:16:59.924Z{"msg":"Crucible stats registered with UUID: ca0281f5-0365-4aad-9661-bd62eccadaea","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.152940247Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32622024-02-06T22:16:59.924Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153001519Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255}
32632024-02-06T22:16:59.924Z{"msg":"ca0281f5-0365-4aad-9661-bd62eccadaea active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153113614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32642024-02-06T22:16:59.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.15314639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32652024-02-06T22:16:59.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153189644Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32662024-02-06T22:16:59.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.15321676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32672024-02-06T22:16:59.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153241987Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32682024-02-06T22:16:59.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153262134Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32692024-02-06T22:16:59.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153285131Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32702024-02-06T22:16:59.924Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153319596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32712024-02-06T22:16:59.924Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153362941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32722024-02-06T22:16:59.924Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153391287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32732024-02-06T22:16:59.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153417723Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32742024-02-06T22:16:59.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153563143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32752024-02-06T22:16:59.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153608167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32762024-02-06T22:16:59.924Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.153982317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32772024-02-06T22:16:59.925Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.15403313Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32782024-02-06T22:16:59.925Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.154160923Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32792024-02-06T22:16:59.925Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.154237452Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32802024-02-06T22:16:59.925Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.154523544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32812024-02-06T22:16:59.925Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.154672144Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32822024-02-06T22:16:59.925Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155078929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32832024-02-06T22:16:59.925Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155122823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32842024-02-06T22:16:59.925Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.15514281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32852024-02-06T22:16:59.925Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155276602Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32862024-02-06T22:16:59.925Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155317037Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32872024-02-06T22:16:59.925Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155343093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32882024-02-06T22:16:59.925Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155375709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32892024-02-06T22:16:59.925Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155410704Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32902024-02-06T22:16:59.925Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.155816939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32912024-02-06T22:16:59.925Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156241281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32922024-02-06T22:16:59.925Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156268858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32932024-02-06T22:16:59.925Z{"msg":"[0]R flush_numbers[0..12]: [60, 60, 52, 30, 53, 58, 60, 56, 57, 57, 47, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156286115Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32942024-02-06T22:16:59.925Z{"msg":"[0]R generation[0..12]: [29, 29, 24, 16, 25, 27, 29, 26, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156321431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32952024-02-06T22:16:59.925Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156347477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32962024-02-06T22:16:59.930Z{"msg":"[1]R flush_numbers[0..12]: [60, 60, 52, 30, 53, 58, 60, 56, 57, 57, 47, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156362395Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32972024-02-06T22:16:59.930Z{"msg":"[1]R generation[0..12]: [29, 29, 24, 16, 25, 27, 29, 26, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156384612Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32982024-02-06T22:16:59.930Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.15640087Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
32992024-02-06T22:16:59.930Z{"msg":"[2]R flush_numbers[0..12]: [60, 60, 52, 30, 53, 58, 60, 56, 57, 57, 47, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156422917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33002024-02-06T22:16:59.930Z{"msg":"[2]R generation[0..12]: [29, 29, 24, 16, 25, 27, 29, 26, 26, 26, 22, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156437745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33012024-02-06T22:16:59.930Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156451983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33022024-02-06T22:16:59.930Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156465381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33032024-02-06T22:16:59.930Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
33042024-02-06T22:16:59.930Z,"time":"2024-02-06T22:16:59.156487868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33052024-02-06T22:16:59.930Z{"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156502716Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33062024-02-06T22:16:59.930Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156516155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33072024-02-06T22:16:59.930Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156536972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33082024-02-06T22:16:59.931Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.15654998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33092024-02-06T22:16:59.931Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156571337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33102024-02-06T22:16:59.931Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156593814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33112024-02-06T22:16:59.931Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156617091Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33122024-02-06T22:16:59.931Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156630809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33132024-02-06T22:16:59.931Z{"msg":"ca0281f5-0365-4aad-9661-bd62eccadaea is now active with session: dbcce7b6-c24d-45de-aca5-27eee60f2d25","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156643797Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33142024-02-06T22:16:59.931Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156664244Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255,"session_id":"dbcce7b6-c24d-45de-aca5-27eee60f2d25"}
33152024-02-06T22:16:59.931Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:16:59.156677513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255}
33162024-02-06T22:16:59.931ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33172024-02-06T22:16:59.931ZLoading write count information from file "/tmp/test_repair_verify.data"
33182024-02-06T22:16:59.931ZRun Repair workload
33192024-02-06T22:16:59.931Z01/30 Write block 460 len 24576 data: 3 3 2 2 2 2
33202024-02-06T22:16:59.931Z02/30 Write block 178 len 40960 data: 2 4 4 5 5 5 5 5 4 4
33212024-02-06T22:16:59.931Z03/30 Read block 82 len 40960
33222024-02-06T22:16:59.931Z04/30 Flush
33232024-02-06T22:16:59.934Z05/30 Write block 209 len 40960 data: 4 3 2 2 2 2 3 3 3 3
33242024-02-06T22:16:59.934Z06/30 Write block 362 len 32768 data: 6 6 5 5 5 5 4 4
33252024-02-06T22:16:59.935Z07/30 Read block 179 len 40960
33262024-02-06T22:16:59.935Z08/30 Write block 454 len 20480 data: 2 2 2 2 2
33272024-02-06T22:16:59.935Z09/30 Write block 355 len 12288 data: 5 5 7
33282024-02-06T22:16:59.935Z10/30 Write block 208 len 24576 data: 4 5 4 3 3 3
33292024-02-06T22:16:59.935Z11/30 Write block 194 len 16384 data: 2 2 2 3
33302024-02-06T22:16:59.935Z12/30 Flush
33312024-02-06T22:16:59.938Z13/30 Write block 31 len 4096 data: 3
33322024-02-06T22:16:59.938Z14/30 Write block 362 len 24576 data: 7 7 6 6 6 6
33332024-02-06T22:16:59.938Z15/30 Write block 231 len 8192 data: 3 3
33342024-02-06T22:16:59.941Z16/30 Read block 179 len 8192
33352024-02-06T22:16:59.941Z17/30 Read block 529 len 12288
33362024-02-06T22:16:59.941Z18/30 Write block 27 len 16384 data: 5 5 4 4
33372024-02-06T22:16:59.941Z19/30 Write block 54 len 32768 data: 8 8 5 5 4 4 4 3
33382024-02-06T22:16:59.944Z20/30 Read block 255 len 12288
33392024-02-06T22:16:59.944Z21/30 Write block 7 len 36864 data: 3 3 4 4 2 2 2 3 4
33402024-02-06T22:16:59.944Z22/30 Write block 177 len 36864 data: 2 3 5 5 6 6 6 6 6
33412024-02-06T22:16:59.948Z23/30 Write block 329 len 40960 data: 3 3 3 3 3 5 5 5 5 6
33422024-02-06T22:16:59.948Z24/30 Write block 2 len 28672 data: 2 3 3 3 3 4 4
33432024-02-06T22:16:59.949Z25/30 Flush
33442024-02-06T22:16:59.951Z26/30 Write block 571 len 28672 data: 5 5 4 4 4 4 4
33452024-02-06T22:16:59.954Z27/30 Read block 404 len 20480
33462024-02-06T22:16:59.954Z28/30 Read block 116 len 12288
33472024-02-06T22:16:59.954Z29/30 Read block 570 len 20480
33482024-02-06T22:16:59.957Z30/30 Write block 531 len 16384 data: 4 4 3 3
33492024-02-06T22:16:59.957Z----------------------------------------------------------------
33502024-02-06T22:16:59.958Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33512024-02-06T22:16:59.958ZStates: Active Active Active
33522024-02-06T22:16:59.958ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33532024-02-06T22:16:59.958Z 1 Acked 1000 Write 6 Done Done Done false
33542024-02-06T22:16:59.958Z 2 Acked 1001 Write 10 Done Done Sent false
33552024-02-06T22:16:59.958Z 3 Acked 1002 Read 10 Done Done Sent false
33562024-02-06T22:16:59.958Z 4 Acked 1003 Flush 0 Done Done Sent false
33572024-02-06T22:16:59.958Z 5 Acked 1004 Write 10 Done Done Sent false
33582024-02-06T22:16:59.958Z 6 Acked 1005 Write 8 Done Done Sent false
33592024-02-06T22:16:59.958Z 7 Acked 1006 Read 10 Done Done Sent false
33602024-02-06T22:16:59.958Z 8 Acked 1007 Write 5 Done Done Sent false
33612024-02-06T22:16:59.958Z 9 Acked 1008 Write 3 Done Done Sent false
33622024-02-06T22:16:59.958Z 10 Acked 1009 Write 6 Done Done Sent false
33632024-02-06T22:16:59.958Z 11 Acked 1010 Write 4 Done Done Sent false
33642024-02-06T22:16:59.958Z 12 Acked 1011 Flush 0 Done Done Sent false
33652024-02-06T22:16:59.958Z 13 Acked 1012 Write 1 Done Done Sent false
33662024-02-06T22:16:59.958Z 14 Acked 1013 Write 6 Done Done Sent false
33672024-02-06T22:16:59.958Z 15 Acked 1014 Write 2 Done Done Sent false
33682024-02-06T22:16:59.958Z 16 Acked 1015 Read 2 Done Done Sent false
33692024-02-06T22:16:59.958Z 17 Acked 1016 Read 3 Done Done Sent false
33702024-02-06T22:16:59.958Z 18 Acked 1017 Write 4 Done Done Sent false
33712024-02-06T22:16:59.958Z 19 Acked 1018 Write 8 Done Done Sent false
33722024-02-06T22:16:59.958Z 20 Acked 1019 Read 3 Done Done Sent false
33732024-02-06T22:16:59.958Z 21 Acked 1020 Write 9 Done Done Sent false
33742024-02-06T22:16:59.958Z 22 Acked 1021 Write 9 Done Done Sent false
33752024-02-06T22:16:59.959Z 23 Acked 1022 Write 10 Done Done Sent false
33762024-02-06T22:16:59.959Z 24 Acked 1023 Write 7 Done Done Sent false
33772024-02-06T22:16:59.959Z 25 Acked 1024 Flush 0 Done Done Sent false
33782024-02-06T22:16:59.959Z 26 Acked 1025 Write 7 Done Done Sent false
33792024-02-06T22:16:59.959Z 27 Acked 1026 Read 5 Done Done Sent false
33802024-02-06T22:16:59.959Z 28 Acked 1027 Read 3 Done Done Sent false
33812024-02-06T22:16:59.959Z 29 Acked 1028 Read 5 Done Done Sent false
33822024-02-06T22:16:59.959Z 30 Acked 1029 Write 4 Sent Sent Sent false
33832024-02-06T22:16:59.959Z STATES DS:0 DS:1 DS:2 TOTAL
33842024-02-06T22:16:59.959Z New 0 0 0 0
33852024-02-06T22:16:59.959Z Sent 1 1 29 31
33862024-02-06T22:16:59.959Z Done 29 29 1 59
33872024-02-06T22:16:59.959Z Skipped 0 0 0 0
33882024-02-06T22:16:59.959Z Error 0 0 0 0
33892024-02-06T22:16:59.959ZLast Flush: 1024 1024 0
33902024-02-06T22:16:59.959ZDownstairs last five completed:
33912024-02-06T22:16:59.959ZUpstairs last five completed: 30 29 28 27 26
33922024-02-06T22:16:59.959Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:16:59.190392147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1255}
33932024-02-06T22:16:59.959ZWrote out file "/tmp/test_repair_verify.data"
33942024-02-06T22:16:59.962Z
33952024-02-06T22:16:59.965ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33962024-02-06T22:16:59.978ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33972024-02-06T22:16:59.979Z 0 000-019 30 30 29 63 63 60 F F F <---
33982024-02-06T22:16:59.979Z 1 020-039 30 30 29 63 63 60 F F F <---
33992024-02-06T22:16:59.979Z 2 040-059 30 30 24 63 63 52 F F F <---
34002024-02-06T22:16:59.979Z 3 060-079 30 30 16 63 63 30 F F F <---
34012024-02-06T22:16:59.979Z 4 080-099 25 25 25 53 53 53 F F F
34022024-02-06T22:16:59.979Z 5 100-119 27 27 27 58 58 58 F F F
34032024-02-06T22:16:59.979Z 6 120-139 29 29 29 60 60 60 F F F
34042024-02-06T22:16:59.979Z 7 140-159 26 26 26 56 56 56 F F F
34052024-02-06T22:16:59.979Z 8 160-179 30 30 26 63 63 57 F F F <---
34062024-02-06T22:16:59.979Z 9 180-199 30 30 26 63 63 57 F F F <---
34072024-02-06T22:16:59.979Z 10 200-219 30 30 22 62 62 47 F F F <---
34082024-02-06T22:16:59.979Z 11 220-239 30 30 26 63 63 57 F F F <---
34092024-02-06T22:16:59.982Z 12 240-259 20 20 20 42 42 42 F F F
34102024-02-06T22:16:59.982Z 13 260-279 28 28 28 59 59 59 F F F
34112024-02-06T22:16:59.982Z 14 280-299 29 29 29 60 60 60 F F F
34122024-02-06T22:16:59.982Z 15 300-319 29 29 29 60 60 60 F F F
34132024-02-06T22:16:59.982Z 16 320-339 30 30 29 63 63 60 F F F <---
34142024-02-06T22:16:59.982Z 17 340-359 30 30 26 62 62 55 F F F <---
34152024-02-06T22:16:59.983Z 18 360-379 30 30 23 63 63 49 F F F <---
34162024-02-06T22:16:59.983Z 19 380-399 24 24 24 52 52 52 F F F
34172024-02-06T22:16:59.983Z 20 400-419 28 28 28 59 59 59 F F F
34182024-02-06T22:16:59.983Z 21 420-439 29 29 29 60 60 60 F F F
34192024-02-06T22:16:59.983Z 22 440-459 30 30 26 62 62 57 F F F <---
34202024-02-06T22:16:59.983Z 23 460-479 30 30 24 61 61 52 F F T <---
34212024-02-06T22:16:59.983Z 24 480-499 26 26 26 54 54 54 F F F
34222024-02-06T22:16:59.983Z 25 500-519 26 26 26 55 55 55 F F F
34232024-02-06T22:16:59.983Z 26 520-539 24 24 24 50 50 50 T T F <---
34242024-02-06T22:16:59.983Z 27 540-559 26 26 26 56 56 56 F F F
34252024-02-06T22:16:59.983Z 28 560-579 29 29 29 60 60 60 T T F <---
34262024-02-06T22:16:59.983Z 29 580-599 29 29 29 60 60 60 F F F
34272024-02-06T22:16:59.983ZMax gen: 30, Max flush: 63
34282024-02-06T22:16:59.983ZError: Difference in extent metadata found!
34292024-02-06T22:16:59.983ZA Difference in extent metadata is expected here
34302024-02-06T22:16:59.983ZOn loop 15
34312024-02-06T22:16:59.983Z
34322024-02-06T22:16:59.986ZVerifying data now
34332024-02-06T22:17:04.704ZNo change after verify
34342024-02-06T22:17:04.705ZLoop: 15 Downstairs dump after verify (and repair):
34352024-02-06T22:17:04.721ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34362024-02-06T22:17:04.722Z 0 000-019 30 30 30 63 63 63 F F F
34372024-02-06T22:17:04.722Z 1 020-039 30 30 30 63 63 63 F F F
34382024-02-06T22:17:04.722Z 2 040-059 30 30 30 63 63 63 F F F
34392024-02-06T22:17:04.722Z 3 060-079 30 30 30 63 63 63 F F F
34402024-02-06T22:17:04.722Z 4 080-099 25 25 25 53 53 53 F F F
34412024-02-06T22:17:04.723Z 5 100-119 27 27 27 58 58 58 F F F
34422024-02-06T22:17:04.723Z 6 120-139 29 29 29 60 60 60 F F F
34432024-02-06T22:17:04.723Z 7 140-159 26 26 26 56 56 56 F F F
34442024-02-06T22:17:04.723Z 8 160-179 30 30 30 63 63 63 F F F
34452024-02-06T22:17:04.729Z 9 180-199 30 30 30 63 63 63 F F F
34462024-02-06T22:17:04.729Z 10 200-219 30 30 30 62 62 62 F F F
34472024-02-06T22:17:04.729Z 11 220-239 30 30 30 63 63 63 F F F
34482024-02-06T22:17:04.729Z 12 240-259 20 20 20 42 42 42 F F F
34492024-02-06T22:17:04.729Z 13 260-279 28 28 28 59 59 59 F F F
34502024-02-06T22:17:04.729Z 14 280-299 29 29 29 60 60 60 F F F
34512024-02-06T22:17:04.729Z 15 300-319 29 29 29 60 60 60 F F F
34522024-02-06T22:17:04.729Z 16 320-339 30 30 30 63 63 63 F F F
34532024-02-06T22:17:04.730Z 17 340-359 30 30 30 62 62 62 F F F
34542024-02-06T22:17:04.730Z 18 360-379 30 30 30 63 63 63 F F F
34552024-02-06T22:17:04.730Z 19 380-399 24 24 24 52 52 52 F F F
34562024-02-06T22:17:04.730Z 20 400-419 28 28 28 59 59 59 F F F
34572024-02-06T22:17:04.730Z 21 420-439 29 29 29 60 60 60 F F F
34582024-02-06T22:17:04.730Z 22 440-459 30 30 30 62 62 62 F F F
34592024-02-06T22:17:04.730Z 23 460-479 30 30 30 61 61 61 F F F
34602024-02-06T22:17:04.731Z 24 480-499 26 26 26 54 54 54 F F F
34612024-02-06T22:17:04.731Z 25 500-519 26 26 26 55 55 55 F F F
34622024-02-06T22:17:04.731Z 26 520-539 31 31 31 64 64 64 F F F
34632024-02-06T22:17:04.731Z 27 540-559 26 26 26 56 56 56 F F F
34642024-02-06T22:17:04.731Z 28 560-579 31 31 31 64 64 64 F F F
34652024-02-06T22:17:04.731Z 29 580-599 29 29 29 60 60 60 F F F
34662024-02-06T22:17:04.731ZMax gen: 31, Max flush: 64
34672024-02-06T22:17:04.731Z
34682024-02-06T22:17:04.731ZBegin loop 16
34692024-02-06T22:17:04.731ZDownstairs to restart: 2
34702024-02-06T22:17:04.746Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969223798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263}
34712024-02-06T22:17:04.746Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
34722024-02-06T22:17:04.746Z,"time":"2024-02-06T22:17:03.969490662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263}
34732024-02-06T22:17:04.746Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969540725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263}
34742024-02-06T22:17:04.746Z{"msg":"Crucible 77b09809-5afc-4241-958f-1497a4926929 has session id: 87a0f56f-243f-48f7-a711-5ce678fc1e35","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969573351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34752024-02-06T22:17:04.747Z{"msg":"Upstairs opts: Upstairs UUID: 77b09809-5afc-4241-958f-1497a4926929, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969632903Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34762024-02-06T22:17:04.747Z{"msg":"Crucible stats registered with UUID: 77b09809-5afc-4241-958f-1497a4926929","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969675407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34772024-02-06T22:17:04.747Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969705623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263}
34782024-02-06T22:17:04.747Z{"msg":"77b09809-5afc-4241-958f-1497a4926929 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969763745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34792024-02-06T22:17:04.747Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969792371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34802024-02-06T22:17:04.747Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969811508Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34812024-02-06T22:17:04.747Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969845004Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34822024-02-06T22:17:04.747Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.96987036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34832024-02-06T22:17:04.747Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969920994Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34842024-02-06T22:17:04.747Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.96995138Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34852024-02-06T22:17:04.747Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.969995154Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34862024-02-06T22:17:04.747Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970033938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34872024-02-06T22:17:04.747Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970077252Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34882024-02-06T22:17:04.747Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:03.970110078Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34892024-02-06T22:17:04.747Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:03.970153092Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34902024-02-06T22:17:04.747Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970199796Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34912024-02-06T22:17:04.748Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:03.970232002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34922024-02-06T22:17:04.748Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970268407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34932024-02-06T22:17:04.748Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970294233Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34942024-02-06T22:17:04.748Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.97031826Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34952024-02-06T22:17:04.748Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970354215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34962024-02-06T22:17:04.749Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.9708333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34972024-02-06T22:17:04.749Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970876924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34982024-02-06T22:17:04.749Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970905591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
34992024-02-06T22:17:04.749Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.970977361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35002024-02-06T22:17:04.749Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.971422771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35012024-02-06T22:17:04.749Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.971560542Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35022024-02-06T22:17:04.749Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.971955869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35032024-02-06T22:17:04.749Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.971990564Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35042024-02-06T22:17:04.750Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.972006742Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35052024-02-06T22:17:04.750Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.972059415Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35062024-02-06T22:17:04.750Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.972099089Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35072024-02-06T22:17:04.750Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:03.972116077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35082024-02-06T22:17:05.742Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.970775366Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35092024-02-06T22:17:05.769Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.97096934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35102024-02-06T22:17:05.769Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.971956646Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35112024-02-06T22:17:05.769Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.971998731Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35122024-02-06T22:17:05.769Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.972473157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35132024-02-06T22:17:05.769Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.972936764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35142024-02-06T22:17:05.770Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.972970789Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35152024-02-06T22:17:05.770Z{"msg":"[0]R flush_numbers[0..12]: [63, 63, 63, 63, 53, 58, 60, 56, 63, 63, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.972986578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35162024-02-06T22:17:05.770Z{"msg":"[0]R generation[0..12]: [30, 30, 30, 30, 25, 27, 29, 26, 30, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973014684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35172024-02-06T22:17:05.770Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973029832Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35182024-02-06T22:17:05.770Z{"msg":"[1]R flush_numbers[0..12]: [63, 63, 63, 63, 53, 58, 60, 56, 63, 63, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973044979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35192024-02-06T22:17:05.770Z{"msg":"[1]R generation[0..12]: [30, 30, 30, 30, 25, 27, 29, 26, 30, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973059278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35202024-02-06T22:17:05.770Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973073606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35212024-02-06T22:17:05.770Z{"msg":"[2]R flush_numbers[0..12]: [63, 63, 63, 63, 53, 58, 60, 56, 63, 63, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973087504Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35222024-02-06T22:17:05.770Z{"msg":"[2]R generation[0..12]: [30, 30, 30, 30, 25, 27, 29, 26, 30, 30, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973109101Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35232024-02-06T22:17:05.770Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973123999Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35242024-02-06T22:17:05.771Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973143956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35252024-02-06T22:17:05.771Z{"msg":"Generation requested: 32 >= found:32","v":0,"Wait for a query_work_queue command to finish before sending IO
35262024-02-06T22:17:05.771Zname":"crucible","level":30,"time":"2024-02-06T22:17:04.973161094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35272024-02-06T22:17:05.771Z{"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973181381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35282024-02-06T22:17:05.771Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973195029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35292024-02-06T22:17:05.771Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973215156Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35302024-02-06T22:17:05.771Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973227965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35312024-02-06T22:17:05.771Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973247982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35322024-02-06T22:17:05.771Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.97326173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35332024-02-06T22:17:05.771Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973275119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35342024-02-06T22:17:05.771Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973294866Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35352024-02-06T22:17:05.771Z{"msg":"77b09809-5afc-4241-958f-1497a4926929 is now active with session: 87a0f56f-243f-48f7-a711-5ce678fc1e35","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973308334Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35362024-02-06T22:17:05.771Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.973320592Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263,"session_id":"87a0f56f-243f-48f7-a711-5ce678fc1e35"}
35372024-02-06T22:17:05.772Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:04.97334013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263}
35382024-02-06T22:17:05.772ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35392024-02-06T22:17:05.772ZLoading write count information from file "/tmp/test_repair_verify.data"
35402024-02-06T22:17:05.772ZRun Repair workload
35412024-02-06T22:17:05.772Z01/30 Write block 168 len 28672 data: 3 3 2 2 3 3 2
35422024-02-06T22:17:05.772Z02/30 Read block 361 len 16384
35432024-02-06T22:17:05.772Z03/30 Write block 483 len 28672 data: 3 3 4 4 2 2 3
35442024-02-06T22:17:05.772Z04/30 Read block 577 len 20480
35452024-02-06T22:17:05.772Z05/30 Read block 475 len 40960
35462024-02-06T22:17:05.772Z06/30 Read block 101 len 28672
35472024-02-06T22:17:05.772Z07/30 Read block 524 len 4096
35482024-02-06T22:17:05.772Z08/30 Flush
35492024-02-06T22:17:05.790Z09/30 Write block 468 len 32768 data: 4 4 4 5 5 4 4 4
35502024-02-06T22:17:05.790Z10/30 Read block 524 len 16384
35512024-02-06T22:17:05.790Z11/30 Flush
35522024-02-06T22:17:05.790Z12/30 Flush
35532024-02-06T22:17:05.790Z13/30 Write block 259 len 28672 data: 2 3 4 4 4 4 4
35542024-02-06T22:17:05.790Z14/30 Read block 270 len 20480
35552024-02-06T22:17:05.790Z15/30 Write block 94 len 8192 data: 2 3
35562024-02-06T22:17:05.791Z16/30 Write block 585 len 16384 data: 2 2 3 3
35572024-02-06T22:17:05.791Z17/30 Flush
35582024-02-06T22:17:05.791Z18/30 Flush
35592024-02-06T22:17:05.791Z19/30 Write block 132 len 32768 data: 2 2 2 3 3 4 4 5
35602024-02-06T22:17:05.791Z20/30 Write block 198 len 28672 data: 2 2 2 2 4 5 5
35612024-02-06T22:17:05.791Z21/30 Read block 515 len 40960
35622024-02-06T22:17:05.792Z22/30 Read block 272 len 12288
35632024-02-06T22:17:05.792Z23/30 Write block 183 len 24576 data: 7 7 7 5 5 4
35642024-02-06T22:17:05.792Z24/30 Write block 290 len 40960 data: 6 6 6 6 6 7 5 5 4 4
35652024-02-06T22:17:05.792Z25/30 Read block 444 len 4096
35662024-02-06T22:17:05.792Z26/30 Write block 346 len 40960 data: 4 4 4 3 5 7 7 5 5 6
35672024-02-06T22:17:05.792Z27/30 Read block 119 len 28672
35682024-02-06T22:17:05.792Z28/30 Read block 471 len 4096
35692024-02-06T22:17:05.793Z29/30 Write block 401 len 16384 data: 4 4 3 3
35702024-02-06T22:17:05.793Z30/30 Read block 24 len 12288
35712024-02-06T22:17:05.793Z----------------------------------------------------------------
35722024-02-06T22:17:05.793Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35732024-02-06T22:17:05.793ZStates: Active Active Active
35742024-02-06T22:17:05.793ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35752024-02-06T22:17:05.793Z 1 Acked 1000 Write 7 Done Done Done false
35762024-02-06T22:17:05.794Z 2 Acked 1001 Read 4 Done Done Done false
35772024-02-06T22:17:05.794Z 3 Acked 1002 Write 7 Done Done Sent false
35782024-02-06T22:17:05.794Z 4 Acked 1003 Read 5 Done Done Sent false
35792024-02-06T22:17:05.794Z 5 Acked 1004 Read 10 Done Done Sent false
35802024-02-06T22:17:05.794Z 6 Acked 1005 Read 7 Done Done Sent false
35812024-02-06T22:17:05.795Z 7 Acked 1006 Read 1 Done Done Sent false
35822024-02-06T22:17:05.795Z 8 Acked 1007 Flush 0 Done Done Sent false
35832024-02-06T22:17:05.795Z 9 Acked 1008 Write 8 Done Done Sent false
35842024-02-06T22:17:05.795Z 10 Acked 1009 Read 4 Done Done Sent false
35852024-02-06T22:17:05.795Z 11 Acked 1010 Flush 0 Done Done Sent false
35862024-02-06T22:17:05.795Z 12 Acked 1011 Flush 0 Done Done Sent false
35872024-02-06T22:17:05.795Z 13 Acked 1012 Write 7 Done Done Sent false
35882024-02-06T22:17:05.795Z 14 Acked 1013 Read 5 Done Done Sent false
35892024-02-06T22:17:05.796Z 15 Acked 1014 Write 2 Done Done Sent false
35902024-02-06T22:17:05.796Z 16 Acked 1015 Write 4 Done Done Sent false
35912024-02-06T22:17:05.796Z 17 Acked 1016 Flush 0 Done Done Sent false
35922024-02-06T22:17:05.796Z 18 Acked 1017 Flush 0 Done Done Sent false
35932024-02-06T22:17:05.796Z 19 Acked 1018 Write 8 Done Done Sent false
35942024-02-06T22:17:05.796Z 20 Acked 1019 Write 7 Done Done Sent false
35952024-02-06T22:17:05.796Z 21 Acked 1020 Read 10 Done Done Sent false
35962024-02-06T22:17:05.796Z 22 Acked 1021 Read 3 Done Done Sent false
35972024-02-06T22:17:05.797Z 23 Acked 1022 Write 6 Done Done Sent false
35982024-02-06T22:17:05.797Z 24 Acked 1023 Write 10 Done Done Sent false
35992024-02-06T22:17:05.797Z 25 Acked 1024 Read 1 Done Done Sent false
36002024-02-06T22:17:05.797Z 26 Acked 1025 Write 10 Done Done Sent false
36012024-02-06T22:17:05.797Z 27 Acked 1026 Read 7 Done Done Sent false
36022024-02-06T22:17:05.797Z 28 Acked 1027 Read 1 Done Done Sent false
36032024-02-06T22:17:05.798Z 29 Acked 1028 Write 4 Done Done Sent false
36042024-02-06T22:17:05.798Z 30 Acked 1029 Read 3 Done Sent Sent false
36052024-02-06T22:17:05.798Z STATES DS:0 DS:1 DS:2 TOTAL
36062024-02-06T22:17:05.798Z New 0 0 0 0
36072024-02-06T22:17:05.798Z Sent 0 1 28 29
36082024-02-06T22:17:05.798Z Done 30 29 2 61
36092024-02-06T22:17:05.798Z Skipped 0 0 0 0
36102024-02-06T22:17:05.799Z Error 0 0 0 0
36112024-02-06T22:17:05.799ZLast Flush: 1017 1017 0
36122024-02-06T22:17:05.799ZDownstairs last five completed:
36132024-02-06T22:17:05.799ZUpstairs last five completed: 30 29 28 27 26
36142024-02-06T22:17:05.799Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:05.004338442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1263}
36152024-02-06T22:17:05.799ZWrote out file "/tmp/test_repair_verify.data"
36162024-02-06T22:17:05.799Z
36172024-02-06T22:17:05.813ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36182024-02-06T22:17:05.814ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36192024-02-06T22:17:05.814Z 0 000-019 30 30 30 63 63 63 F F F
36202024-02-06T22:17:05.814Z 1 020-039 30 30 30 63 63 63 F F F
36212024-02-06T22:17:05.814Z 2 040-059 30 30 30 63 63 63 F F F
36222024-02-06T22:17:05.814Z 3 060-079 30 30 30 63 63 63 F F F
36232024-02-06T22:17:05.814Z 4 080-099 32 32 25 68 68 53 F F F <---
36242024-02-06T22:17:05.814Z 5 100-119 27 27 27 58 58 58 F F F
36252024-02-06T22:17:05.815Z 6 120-139 29 29 29 60 60 60 T T F <---
36262024-02-06T22:17:05.815Z 7 140-159 26 26 26 56 56 56 F F F
36272024-02-06T22:17:05.815Z 8 160-179 32 32 30 65 65 63 F F T <---
36282024-02-06T22:17:05.815Z 9 180-199 30 30 30 63 63 63 T T F <---
36292024-02-06T22:17:05.815Z 10 200-219 30 30 30 62 62 62 T T F <---
36302024-02-06T22:17:05.815Z 11 220-239 30 30 30 63 63 63 F F F
36312024-02-06T22:17:05.815Z 12 240-259 32 32 20 68 68 42 F F F <---
36322024-02-06T22:17:05.815Z 13 260-279 32 32 28 68 68 59 F F F <---
36332024-02-06T22:17:05.816Z 14 280-299 29 29 29 60 60 60 T T F <---
36342024-02-06T22:17:05.816Z 15 300-319 29 29 29 60 60 60 F F F
36352024-02-06T22:17:05.816Z 16 320-339 30 30 30 63 63 63 F F F
36362024-02-06T22:17:05.816Z 17 340-359 30 30 30 62 62 62 T T F <---
36372024-02-06T22:17:05.816Z 18 360-379 30 30 30 63 63 63 F F F
36382024-02-06T22:17:05.816Z 19 380-399 24 24 24 52 52 52 F F F
36392024-02-06T22:17:05.816Z 20 400-419 28 28 28 59 59 59 T T F <---
36402024-02-06T22:17:05.816Z 21 420-439 29 29 29 60 60 60 F F F
36412024-02-06T22:17:05.817Z 22 440-459 30 30 30 62 62 62 F F F
36422024-02-06T22:17:05.817Z 23 460-479 32 32 30 66 66 61 F F F <---
36432024-02-06T22:17:05.817Z 24 480-499 32 32 26 65 65 54 F F F <---
36442024-02-06T22:17:05.817Z 25 500-519 26 26 26 55 55 55 F F F
36452024-02-06T22:17:05.817Z 26 520-539 31 31 31 64 64 64 F F F
36462024-02-06T22:17:05.817Z 27 540-559 26 26 26 56 56 56 F F F
36472024-02-06T22:17:05.817Z 28 560-579 31 31 31 64 64 64 F F F
36482024-02-06T22:17:05.817Z 29 580-599 32 32 29 68 68 60 F F F <---
36492024-02-06T22:17:05.818ZMax gen: 32, Max flush: 68
36502024-02-06T22:17:05.818ZError: Difference in extent metadata found!
36512024-02-06T22:17:05.818ZA Difference in extent metadata is expected here
36522024-02-06T22:17:05.818ZOn loop 16
36532024-02-06T22:17:05.818Z
36542024-02-06T22:17:05.818ZVerifying data now
36552024-02-06T22:17:10.458ZNo change after verify
36562024-02-06T22:17:10.458ZLoop: 16 Downstairs dump after verify (and repair):
36572024-02-06T22:17:10.885ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36582024-02-06T22:17:10.885Z 0 000-019 30 30 30 63 63 63 F F F
36592024-02-06T22:17:10.885Z 1 020-039 30 30 30 63 63 63 F F F
36602024-02-06T22:17:10.885Z 2 040-059 30 30 30 63 63 63 F F F
36612024-02-06T22:17:10.885Z 3 060-079 30 30 30 63 63 63 F F F
36622024-02-06T22:17:10.885Z 4 080-099 32 32 32 68 68 68 F F F
36632024-02-06T22:17:10.885Z 5 100-119 27 27 27 58 58 58 F F F
36642024-02-06T22:17:10.885Z 6 120-139 33 33 33 69 69 69 F F F
36652024-02-06T22:17:10.885Z 7 140-159 26 26 26 56 56 56 F F F
36662024-02-06T22:17:10.893Z 8 160-179 32 32 32 65 65 65 F F F
36672024-02-06T22:17:10.893Z 9 180-199 33 33 33 69 69 69 F F F
36682024-02-06T22:17:10.893Z 10 200-219 33 33 33 69 69 69 F F F
36692024-02-06T22:17:10.893Z 11 220-239 30 30 30 63 63 63 F F F
36702024-02-06T22:17:10.893Z 12 240-259 32 32 32 68 68 68 F F F
36712024-02-06T22:17:10.893Z 13 260-279 32 32 32 68 68 68 F F F
36722024-02-06T22:17:10.893Z 14 280-299 33 33 33 69 69 69 F F F
36732024-02-06T22:17:10.893Z 15 300-319 29 29 29 60 60 60 F F F
36742024-02-06T22:17:10.893Z 16 320-339 30 30 30 63 63 63 F F F
36752024-02-06T22:17:10.893Z 17 340-359 33 33 33 69 69 69 F F F
36762024-02-06T22:17:10.893Z 18 360-379 30 30 30 63 63 63 F F F
36772024-02-06T22:17:10.893Z 19 380-399 24 24 24 52 52 52 F F F
36782024-02-06T22:17:10.894Z 20 400-419 33 33 33 69 69 69 F F F
36792024-02-06T22:17:10.894Z 21 420-439 29 29 29 60 60 60 F F F
36802024-02-06T22:17:10.894Z 22 440-459 30 30 30 62 62 62 F F F
36812024-02-06T22:17:10.894Z 23 460-479 32 32 32 66 66 66 F F F
36822024-02-06T22:17:10.894Z 24 480-499 32 32 32 65 65 65 F F F
36832024-02-06T22:17:10.894Z 25 500-519 26 26 26 55 55 55 F F F
36842024-02-06T22:17:10.894Z 26 520-539 31 31 31 64 64 64 F F F
36852024-02-06T22:17:10.894Z 27 540-559 26 26 26 56 56 56 F F F
36862024-02-06T22:17:10.894Z 28 560-579 31 31 31 64 64 64 F F F
36872024-02-06T22:17:10.894Z 29 580-599 32 32 32 68 68 68 F F F
36882024-02-06T22:17:10.894ZMax gen: 33, Max flush: 69
36892024-02-06T22:17:10.894Z
36902024-02-06T22:17:10.894ZBegin loop 17
36912024-02-06T22:17:10.894ZDownstairs to restart: 1
36922024-02-06T22:17:10.894Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.727890692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271}
36932024-02-06T22:17:10.894Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728146467Z","hostname":"Crucible runtime is spawned
36942024-02-06T22:17:10.894Zip-10-150-1-27.us-west-2.compute.internal","pid":1271}
36952024-02-06T22:17:10.894Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728191741Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271}
36962024-02-06T22:17:10.894Z{"msg":"Crucible 92005323-a19f-41df-b08e-8760c9ea520d has session id: 1d79ed91-a876-4781-8765-d2fd4b5942c8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728215688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
36972024-02-06T22:17:10.894Z{"msg":"Upstairs opts: Upstairs UUID: 92005323-a19f-41df-b08e-8760c9ea520d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728258552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
36982024-02-06T22:17:10.894Z{"msg":"Crucible stats registered with UUID: 92005323-a19f-41df-b08e-8760c9ea520d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728276639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
36992024-02-06T22:17:10.894Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728310105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271}
37002024-02-06T22:17:10.894Z{"msg":"92005323-a19f-41df-b08e-8760c9ea520d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728414431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37012024-02-06T22:17:10.895Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728463844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37022024-02-06T22:17:10.895Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.72849587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37032024-02-06T22:17:10.895Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728544193Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37042024-02-06T22:17:10.895Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728578189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37052024-02-06T22:17:10.895Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728595456Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37062024-02-06T22:17:10.895Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.72864236Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37072024-02-06T22:17:10.895Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728722759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37082024-02-06T22:17:10.895Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728764203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37092024-02-06T22:17:10.895Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.72879225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37102024-02-06T22:17:10.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728841533Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37112024-02-06T22:17:10.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728883447Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37122024-02-06T22:17:10.895Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.728924722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37132024-02-06T22:17:10.895Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.729412146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37142024-02-06T22:17:10.895Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.72945538Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37152024-02-06T22:17:10.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.729497504Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37162024-02-06T22:17:10.896Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.729522721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37172024-02-06T22:17:10.896Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730166864Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37182024-02-06T22:17:10.896Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730219427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37192024-02-06T22:17:10.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730578618Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37202024-02-06T22:17:10.896Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730629151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37212024-02-06T22:17:10.896Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.7307871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37222024-02-06T22:17:10.896Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730820246Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37232024-02-06T22:17:10.896Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730836003Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37242024-02-06T22:17:10.896Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730848982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37252024-02-06T22:17:10.896Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730876788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37262024-02-06T22:17:10.896Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.730892036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37272024-02-06T22:17:10.896Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731115586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37282024-02-06T22:17:10.896Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.73152858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37292024-02-06T22:17:10.897Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731588092Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37302024-02-06T22:17:10.897Z{"msg":"[0]R flush_numbers[0..12]: [63, 63, 63, 63, 68, 58, 69, 56, 65, 69, 69, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731633956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37312024-02-06T22:17:10.897Z{"msg":"[0]R generation[0..12]: [30, 30, 30, 30, 32, 27, 33, 26, 32, 33, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731670021Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37322024-02-06T22:17:10.897Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731696197Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37332024-02-06T22:17:10.897Z{"msg":"[1]R flush_numbers[0..12]: [63, 63, 63, 63, 68, 58, 69, 56, 65, 69, 69, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731733992Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37342024-02-06T22:17:10.897ZWait for a query_work_queue command to finish before sending IO
37352024-02-06T22:17:10.897Z{"msg":"[1]R generation[0..12]: [30, 30, 30, 30, 32, 27, 33, 26, 32, 33, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731778436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37362024-02-06T22:17:10.897Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731820141Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37372024-02-06T22:17:10.897Z{"msg":"[2]R flush_numbers[0..12]: [63, 63, 63, 63, 68, 58, 69, 56, 65, 69, 69, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731860995Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37382024-02-06T22:17:10.897Z{"msg":"[2]R generation[0..12]: [30, 30, 30, 30, 32, 27, 33, 26, 32, 33, 33, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.7318977Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37392024-02-06T22:17:10.897Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731933375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37402024-02-06T22:17:10.897Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.731977019Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37412024-02-06T22:17:10.898Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732031122Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37422024-02-06T22:17:10.898Z{"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.73204955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37432024-02-06T22:17:10.898Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732071756Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37442024-02-06T22:17:10.898Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732086225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37452024-02-06T22:17:10.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732099123Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37462024-02-06T22:17:10.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732112791Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37472024-02-06T22:17:10.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732133868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37482024-02-06T22:17:10.898Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732147666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37492024-02-06T22:17:10.898Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732167384Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37502024-02-06T22:17:10.898Z{"msg":"92005323-a19f-41df-b08e-8760c9ea520d is now active with session: 1d79ed91-a876-4781-8765-d2fd4b5942c8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732181012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37512024-02-06T22:17:10.898Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37522024-02-06T22:17:10.899Z,"time":"2024-02-06T22:17:09.732200079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271,"session_id":"1d79ed91-a876-4781-8765-d2fd4b5942c8"}
37532024-02-06T22:17:10.899Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:09.732215957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271}
37542024-02-06T22:17:10.899ZLoading write count information from file "/tmp/test_repair_verify.data"
37552024-02-06T22:17:10.899ZRun Repair workload
37562024-02-06T22:17:10.899Z01/30 Flush
37572024-02-06T22:17:10.899Z02/30 Write block 107 len 16384 data: 3 4 3 3
37582024-02-06T22:17:10.899Z03/30 Read block 66 len 24576
37592024-02-06T22:17:10.899Z04/30 Read block 498 len 16384
37602024-02-06T22:17:10.899Z05/30 Read block 72 len 20480
37612024-02-06T22:17:10.899Z06/30 Write block 538 len 36864 data: 2 2 2 4 4 4 4 4 4
37622024-02-06T22:17:10.899Z07/30 Write block 248 len 20480 data: 5 6 5 4 4
37632024-02-06T22:17:10.899Z08/30 Flush
37642024-02-06T22:17:10.899Z09/30 Write block 14 len 40960 data: 4 5 4 3 3 4 3 3 4 4
37652024-02-06T22:17:10.899Z10/30 Write block 442 len 40960 data: 4 3 2 2 2 2 3 3 3 3
37662024-02-06T22:17:10.905Z11/30 Flush
37672024-02-06T22:17:10.905Z12/30 Write block 67 len 36864 data: 3 4 4 4 3 3 3 5 4
37682024-02-06T22:17:10.905Z13/30 Read block 195 len 36864
37692024-02-06T22:17:10.905Z14/30 Write block 424 len 20480 data: 3 3 4 4 5
37702024-02-06T22:17:10.905Z15/30 Read block 486 len 4096
37712024-02-06T22:17:10.905Z16/30 Write block 445 len 28672 data: 3 3 3 4 4 4 4
37722024-02-06T22:17:10.905Z17/30 Read block 289 len 16384
37732024-02-06T22:17:10.905Z18/30 Flush
37742024-02-06T22:17:10.905Z19/30 Write block 176 len 24576 data: 2 3 4 6 6 7
37752024-02-06T22:17:10.906Z20/30 Read block 182 len 36864
37762024-02-06T22:17:10.906Z21/30 Flush
37772024-02-06T22:17:10.906Z22/30 Write block 247 len 4096 data: 5
37782024-02-06T22:17:10.906Z23/30 Write block 336 len 20480 data: 6 6 7 5 5
37792024-02-06T22:17:10.906Z24/30 Flush
37802024-02-06T22:17:10.906Z25/30 Write block 127 len 4096 data: 7
37812024-02-06T22:17:10.906Z26/30 Write block 107 len 40960 data: 4 5 4 4 3 4 4 5 6 5
37822024-02-06T22:17:10.906Z27/30 Write block 408 len 16384 data: 2 3 3 3
37832024-02-06T22:17:10.906Z28/30 Write block 223 len 16384 data: 3 3 5 5
37842024-02-06T22:17:10.906Z29/30 Write block 551 len 32768 data: 3 3 3 3 2 3 3 4
37852024-02-06T22:17:10.906Z30/30 Write block 571 len 32768 data: 6 6 5 5 5 5 5 4
37862024-02-06T22:17:10.906Z----------------------------------------------------------------
37872024-02-06T22:17:10.906Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37882024-02-06T22:17:10.907ZStates: Active Active Active
37892024-02-06T22:17:10.907ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37902024-02-06T22:17:10.907Z 1 Acked 1000 Flush 0 Done Sent Done false
37912024-02-06T22:17:10.907Z 2 Acked 1001 Write 4 Done Sent Done false
37922024-02-06T22:17:10.907Z 3 Acked 1002 Read 6 Done Sent Done false
37932024-02-06T22:17:10.907Z 4 Acked 1003 Read 4 Done Sent Done false
37942024-02-06T22:17:10.907Z 5 Acked 1004 Read 5 Done Sent Done false
37952024-02-06T22:17:10.907Z 6 Acked 1005 Write 9 Done Sent Done false
37962024-02-06T22:17:10.907Z 7 Acked 1006 Write 5 Done Sent Done false
37972024-02-06T22:17:10.907Z 8 Acked 1007 Flush 0 Done Sent Done false
37982024-02-06T22:17:10.907Z 9 Acked 1008 Write 10 Done Sent Done false
37992024-02-06T22:17:10.907Z 10 Acked 1009 Write 10 Done Sent Done false
38002024-02-06T22:17:10.907Z 11 Acked 1010 Flush 0 Done Sent Done false
38012024-02-06T22:17:10.907Z 12 Acked 1011 Write 9 Done Sent Done false
38022024-02-06T22:17:10.908Z 13 Acked 1012 Read 9 Done Sent Done false
38032024-02-06T22:17:10.908Z 14 Acked 1013 Write 5 Done Sent Done false
38042024-02-06T22:17:10.908Z 15 Acked 1014 Read 1 Done Sent Done false
38052024-02-06T22:17:10.908Z 16 Acked 1015 Write 7 Done Sent Done false
38062024-02-06T22:17:10.908Z 17 Acked 1016 Read 4 Done Sent Done false
38072024-02-06T22:17:10.908Z 18 Acked 1017 Flush 0 Done Sent Done false
38082024-02-06T22:17:10.908Z 19 Acked 1018 Write 6 Done Sent Done false
38092024-02-06T22:17:10.908Z 20 Acked 1019 Read 9 Done Sent Done false
38102024-02-06T22:17:10.908Z 21 Acked 1020 Flush 0 Done Sent Done false
38112024-02-06T22:17:10.908Z 22 Acked 1021 Write 1 Done Sent Done false
38122024-02-06T22:17:10.908Z 23 Acked 1022 Write 5 Done Sent Done false
38132024-02-06T22:17:10.908Z 24 Acked 1023 Flush 0 Done Sent Done false
38142024-02-06T22:17:10.908Z 25 Acked 1024 Write 1 Done Sent Done false
38152024-02-06T22:17:10.909Z 26 Acked 1025 Write 10 Done Sent Done false
38162024-02-06T22:17:10.909Z 27 Acked 1026 Write 4 Done Sent Done false
38172024-02-06T22:17:10.909Z 28 Acked 1027 Write 4 Done Sent Done false
38182024-02-06T22:17:10.909Z 29 Acked 1028 Write 8 Done Sent Sent false
38192024-02-06T22:17:10.909Z 30 Acked 1029 Write 8 Sent Sent Sent false
38202024-02-06T22:17:10.909Z STATES DS:0 DS:1 DS:2 TOTAL
38212024-02-06T22:17:10.909Z New 0 0 0 0
38222024-02-06T22:17:10.909Z Sent 1 30 2 33
38232024-02-06T22:17:10.909Z Done 29 0 28 57
38242024-02-06T22:17:10.910Z Skipped 0 0 0 0
38252024-02-06T22:17:10.910Z Error 0 0 0 0
38262024-02-06T22:17:10.910ZLast Flush: 1023 0 1023
38272024-02-06T22:17:10.910ZDownstairs last five completed:
38282024-02-06T22:17:10.910ZUpstairs last five completed: 30 29 28 27 26
38292024-02-06T22:17:10.910Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:09.765391915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1271}
38302024-02-06T22:17:10.910ZWrote out file "/tmp/test_repair_verify.data"
38312024-02-06T22:17:10.910Z
38322024-02-06T22:17:10.910ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38332024-02-06T22:17:10.910ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38342024-02-06T22:17:10.910Z 0 000-019 34 30 34 72 63 72 F F F <---
38352024-02-06T22:17:10.910Z 1 020-039 34 30 34 72 63 72 F F F <---
38362024-02-06T22:17:10.910Z 2 040-059 30 30 30 63 63 63 F F F
38372024-02-06T22:17:10.910Z 3 060-079 34 30 34 73 63 73 F F F <---
38382024-02-06T22:17:10.910Z 4 080-099 32 32 32 68 68 68 F F F
38392024-02-06T22:17:10.910Z 5 100-119 34 27 34 71 58 71 T F T <---
38402024-02-06T22:17:10.911Z 6 120-139 33 33 33 69 69 69 T F T <---
38412024-02-06T22:17:10.911Z 7 140-159 26 26 26 56 56 56 F F F
38422024-02-06T22:17:10.911Z 8 160-179 34 32 34 74 65 74 F F F <---
38432024-02-06T22:17:10.911Z 9 180-199 34 33 34 74 69 74 F F F <---
38442024-02-06T22:17:10.911Z 10 200-219 33 33 33 69 69 69 F F F
38452024-02-06T22:17:10.911Z 11 220-239 30 30 30 63 63 63 T F T <---
38462024-02-06T22:17:10.911Z 12 240-259 34 32 34 75 68 75 F F F <---
38472024-02-06T22:17:10.911Z 13 260-279 32 32 32 68 68 68 F F F
38482024-02-06T22:17:10.911Z 14 280-299 33 33 33 69 69 69 F F F
38492024-02-06T22:17:10.911Z 15 300-319 29 29 29 60 60 60 F F F
38502024-02-06T22:17:10.911Z 16 320-339 34 30 34 75 63 75 F F F <---
38512024-02-06T22:17:10.911Z 17 340-359 34 33 34 75 69 75 F F F <---
38522024-02-06T22:17:10.911Z 18 360-379 30 30 30 63 63 63 F F F
38532024-02-06T22:17:10.911Z 19 380-399 24 24 24 52 52 52 F F F
38542024-02-06T22:17:10.912Z 20 400-419 33 33 33 69 69 69 T F T <---
38552024-02-06T22:17:10.912Z 21 420-439 34 29 34 73 60 73 F F F <---
38562024-02-06T22:17:10.912Z 22 440-459 34 30 34 73 62 73 F F F <---
38572024-02-06T22:17:10.912Z 23 460-479 32 32 32 66 66 66 F F F
38582024-02-06T22:17:10.912Z 24 480-499 32 32 32 65 65 65 F F F
38592024-02-06T22:17:10.912Z 25 500-519 26 26 26 55 55 55 F F F
38602024-02-06T22:17:10.912Z 26 520-539 34 31 34 71 64 71 F F F <---
38612024-02-06T22:17:10.912Z 27 540-559 34 26 34 71 56 71 T F T <---
38622024-02-06T22:17:10.912Z 28 560-579 31 31 31 64 64 64 T F T <---
38632024-02-06T22:17:10.912Z 29 580-599 32 32 32 68 68 68 F F F
38642024-02-06T22:17:10.912ZMax gen: 34, Max flush: 75
38652024-02-06T22:17:10.912ZError: Difference in extent metadata found!
38662024-02-06T22:17:10.916ZA Difference in extent metadata is expected here
38672024-02-06T22:17:10.916ZOn loop 17
38682024-02-06T22:17:10.917Z
38692024-02-06T22:17:10.917ZVerifying data now
38702024-02-06T22:17:15.359ZNo change after verify
38712024-02-06T22:17:15.359ZLoop: 17 Downstairs dump after verify (and repair):
38722024-02-06T22:17:15.374ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38732024-02-06T22:17:15.374Z 0 000-019 34 34 34 72 72 72 F F F
38742024-02-06T22:17:15.374Z 1 020-039 34 34 34 72 72 72 F F F
38752024-02-06T22:17:15.374Z 2 040-059 30 30 30 63 63 63 F F F
38762024-02-06T22:17:15.374Z 3 060-079 34 34 34 73 73 73 F F F
38772024-02-06T22:17:15.374Z 4 080-099 32 32 32 68 68 68 F F F
38782024-02-06T22:17:15.375Z 5 100-119 35 35 35 76 76 76 F F F
38792024-02-06T22:17:15.375Z 6 120-139 35 35 35 76 76 76 F F F
38802024-02-06T22:17:15.375Z 7 140-159 26 26 26 56 56 56 F F F
38812024-02-06T22:17:15.375Z 8 160-179 34 34 34 74 74 74 F F F
38822024-02-06T22:17:15.375Z 9 180-199 34 34 34 74 74 74 F F F
38832024-02-06T22:17:15.375Z 10 200-219 33 33 33 69 69 69 F F F
38842024-02-06T22:17:15.375Z 11 220-239 35 35 35 76 76 76 F F F
38852024-02-06T22:17:15.375Z 12 240-259 34 34 34 75 75 75 F F F
38862024-02-06T22:17:15.378Z 13 260-279 32 32 32 68 68 68 F F F
38872024-02-06T22:17:15.378Z 14 280-299 33 33 33 69 69 69 F F F
38882024-02-06T22:17:15.378Z 15 300-319 29 29 29 60 60 60 F F F
38892024-02-06T22:17:15.379Z 16 320-339 34 34 34 75 75 75 F F F
38902024-02-06T22:17:15.379Z 17 340-359 34 34 34 75 75 75 F F F
38912024-02-06T22:17:15.379Z 18 360-379 30 30 30 63 63 63 F F F
38922024-02-06T22:17:15.379Z 19 380-399 24 24 24 52 52 52 F F F
38932024-02-06T22:17:15.379Z 20 400-419 35 35 35 76 76 76 F F F
38942024-02-06T22:17:15.379Z 21 420-439 34 34 34 73 73 73 F F F
38952024-02-06T22:17:15.379Z 22 440-459 34 34 34 73 73 73 F F F
38962024-02-06T22:17:15.379Z 23 460-479 32 32 32 66 66 66 F F F
38972024-02-06T22:17:15.379Z 24 480-499 32 32 32 65 65 65 F F F
38982024-02-06T22:17:15.379Z 25 500-519 26 26 26 55 55 55 F F F
38992024-02-06T22:17:15.379Z 26 520-539 34 34 34 71 71 71 F F F
39002024-02-06T22:17:15.379Z 27 540-559 35 35 35 76 76 76 F F F
39012024-02-06T22:17:15.379Z 28 560-579 35 35 35 76 76 76 F F F
39022024-02-06T22:17:15.379Z 29 580-599 32 32 32 68 68 68 F F F
39032024-02-06T22:17:15.379ZMax gen: 35, Max flush: 76
39042024-02-06T22:17:15.379Z
39052024-02-06T22:17:15.379ZBegin loop 18
39062024-02-06T22:17:15.380ZDownstairs to restart: 1
39072024-02-06T22:17:15.394Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.626374636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279}
39082024-02-06T22:17:15.394Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.626611554Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279}
39092024-02-06T22:17:15.394Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.626629221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279}
39102024-02-06T22:17:15.394Z{"msg":"Crucible 96b91b58-98fa-4f70-82a4-93914355a696 has session id: aa513370-22d4-4781-94c6-5f78e660a205","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.626651768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39112024-02-06T22:17:15.399Z{"msg":"Upstairs opts: Upstairs UUID: 96b91b58-98fa-4f70-82a4-93914355a696, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
39122024-02-06T22:17:15.399Z,"time":"2024-02-06T22:17:14.626694063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39132024-02-06T22:17:15.399Z{"msg":"Crucible stats registered with UUID: 96b91b58-98fa-4f70-82a4-93914355a696","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.626734427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39142024-02-06T22:17:15.399Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.62678528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279}
39152024-02-06T22:17:15.399Z{"msg":"96b91b58-98fa-4f70-82a4-93914355a696 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.626928421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39162024-02-06T22:17:15.399Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.626960017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39172024-02-06T22:17:15.399Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.62701152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39182024-02-06T22:17:15.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627036876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39192024-02-06T22:17:15.400Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627061093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39202024-02-06T22:17:15.400Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627093729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39212024-02-06T22:17:15.400Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627119245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39222024-02-06T22:17:15.400Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.62716052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39232024-02-06T22:17:15.400Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627197635Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39242024-02-06T22:17:15.400Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627252937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39252024-02-06T22:17:15.400Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627293422Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39262024-02-06T22:17:15.400Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627321978Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39272024-02-06T22:17:15.400Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627349774Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39282024-02-06T22:17:15.400Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627939315Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39292024-02-06T22:17:15.400Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.627983198Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39302024-02-06T22:17:15.400Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.628021893Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39312024-02-06T22:17:15.400Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.628065437Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39322024-02-06T22:17:15.400Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.62856763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39332024-02-06T22:17:15.400Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.628614143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39342024-02-06T22:17:15.400Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.629217002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39352024-02-06T22:17:15.400Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.629263665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39362024-02-06T22:17:15.400Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.629285122Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39372024-02-06T22:17:15.401Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.6293013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39382024-02-06T22:17:15.401Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.629327617Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39392024-02-06T22:17:15.401Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.629343695Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39402024-02-06T22:17:15.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.629357333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39412024-02-06T22:17:15.401Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.62937931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39422024-02-06T22:17:15.401Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.62982172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39432024-02-06T22:17:15.406Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630261091Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39442024-02-06T22:17:15.406Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630307914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39452024-02-06T22:17:15.406Z{"msg":"[0]R flush_numbers[0..12]: [72, 72, 63, 73, 68, 76, 76, 56, 74, 74, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630333621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39462024-02-06T22:17:15.406Z{"msg":"[0]R generation[0..12]: [34, 34, 30, 34, 32, 35, 35, 26, 34, 34, 33, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630368876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39472024-02-06T22:17:15.406Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630404761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39482024-02-06T22:17:15.406Z{"msg":"[1]R flush_numbers[0..12]: [72, 72, 63, 73, 68, 76, 76, 56, 74, 74, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630440797Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39492024-02-06T22:17:15.406Z{"msg":"[1]R generation[0..12]: [34, 34, 30, 34, 32, 35, 35, 26, 34, 34, 33, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630477801Z"Wait for a query_work_queue command to finish before sending IO
39502024-02-06T22:17:15.406Z,"hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39512024-02-06T22:17:15.406Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630524395Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39522024-02-06T22:17:15.406Z{"msg":"[2]R flush_numbers[0..12]: [72, 72, 63, 73, 68, 76, 76, 56, 74, 74, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.63056543Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39532024-02-06T22:17:15.406Z{"msg":"[2]R generation[0..12]: [34, 34, 30, 34, 32, 35, 35, 26, 34, 34, 33, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630602684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39542024-02-06T22:17:15.406Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630640619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39552024-02-06T22:17:15.407Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630676834Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39562024-02-06T22:17:15.407Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.63071291Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39572024-02-06T22:17:15.407Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630748735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39582024-02-06T22:17:15.407Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630775651Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39592024-02-06T22:17:15.407Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630812076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39602024-02-06T22:17:15.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630845892Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39612024-02-06T22:17:15.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630871818Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pidRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39622024-02-06T22:17:15.407Z":1279,"client":"1","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39632024-02-06T22:17:15.407Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630908913Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39642024-02-06T22:17:15.407Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630925611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39652024-02-06T22:17:15.407Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630947518Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39662024-02-06T22:17:15.407Z{"msg":"96b91b58-98fa-4f70-82a4-93914355a696 is now active with session: aa513370-22d4-4781-94c6-5f78e660a205","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630961666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39672024-02-06T22:17:15.407Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630974174Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279,"session_id":"aa513370-22d4-4781-94c6-5f78e660a205"}
39682024-02-06T22:17:15.407Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:14.630994702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279}
39692024-02-06T22:17:15.407ZLoading write count information from file "/tmp/test_repair_verify.data"
39702024-02-06T22:17:15.407ZRun Repair workload
39712024-02-06T22:17:15.407Z01/30 Write block 113 len 8192 data: 5 6
39722024-02-06T22:17:15.407Z02/30 Read block 430 len 8192
39732024-02-06T22:17:15.407Z03/30 Write block 589 len 12288 data: 4 6 6
39742024-02-06T22:17:15.408Z04/30 Write block 142 len 20480 data: 3 5 5 6 6
39752024-02-06T22:17:15.408Z05/30 Read block 35 len 24576
39762024-02-06T22:17:15.408Z06/30 Write block 64 len 24576 data: 3 3 3 4 5 5
39772024-02-06T22:17:15.408Z07/30 Read block 544 len 4096
39782024-02-06T22:17:15.412Z08/30 Write block 355 len 32768 data: 7 6 8 6 6 6 6 8
39792024-02-06T22:17:15.412Z09/30 Write block 209 len 4096 data: 6
39802024-02-06T22:17:15.412Z10/30 Read block 472 len 4096
39812024-02-06T22:17:15.412Z11/30 Write block 334 len 36864 data: 6 6 7 7 8 6 6 4 5
39822024-02-06T22:17:15.412Z12/30 Read block 565 len 24576
39832024-02-06T22:17:15.412Z13/30 Write block 576 len 12288 data: 6 6 5
39842024-02-06T22:17:15.412Z14/30 Flush
39852024-02-06T22:17:15.415Z15/30 Write block 515 len 8192 data: 4 4
39862024-02-06T22:17:15.415Z16/30 Read block 94 len 24576
39872024-02-06T22:17:15.420Z17/30 Read block 468 len 36864
39882024-02-06T22:17:15.420Z18/30 Read block 210 len 36864
39892024-02-06T22:17:15.420Z19/30 Write block 147 len 36864 data: 7 9 8 7 5 5 7 5 6
39902024-02-06T22:17:15.424Z20/30 Write block 146 len 36864 data: 7 8 10 9 8 6 6 8 6
39912024-02-06T22:17:15.424Z21/30 Read block 0 len 32768
39922024-02-06T22:17:15.424Z22/30 Flush
39932024-02-06T22:17:15.427Z23/30 Flush
39942024-02-06T22:17:15.427Z24/30 Write block 563 len 24576 data: 2 3 3 2 5 5
39952024-02-06T22:17:15.427Z25/30 Write block 53 len 36864 data: 9 9 9 6 6 5 5 5 4
39962024-02-06T22:17:15.427Z26/30 Write block 202 len 36864 data: 5 6 6 5 5 4 5 7 5
39972024-02-06T22:17:15.431Z27/30 Read block 13 len 36864
39982024-02-06T22:17:15.431Z28/30 Write block 157 len 36864 data: 5 5 5 4 4 3 4 4 3
39992024-02-06T22:17:15.431Z29/30 Write block 278 len 32768 data: 3 4 4 4 4 4 4 3
40002024-02-06T22:17:15.431Z30/30 Read block 261 len 16384
40012024-02-06T22:17:15.435Z----------------------------------------------------------------
40022024-02-06T22:17:15.435Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
40032024-02-06T22:17:15.435ZStates: Active Active Active
40042024-02-06T22:17:15.435ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40052024-02-06T22:17:15.435Z 1 Acked 1000 Write 2 Done Done Done false
40062024-02-06T22:17:15.435Z 2 Acked 1001 Read 2 Done Done Done false
40072024-02-06T22:17:15.435Z 3 Acked 1002 Write 3 Done Done Done false
40082024-02-06T22:17:15.435Z 4 Acked 1003 Write 5 Done Done Done false
40092024-02-06T22:17:15.435Z 5 Acked 1004 Read 6 Done Done Done false
40102024-02-06T22:17:15.435Z 6 Acked 1005 Write 6 Done Done Done false
40112024-02-06T22:17:15.435Z 7 Acked 1006 Read 1 Done Done Done false
40122024-02-06T22:17:15.436Z 8 Acked 1007 Write 8 Done Done Done false
40132024-02-06T22:17:15.436Z 9 Acked 1008 Write 1 Done Done Done false
40142024-02-06T22:17:15.436Z 10 Acked 1009 Read 1 Done Done Done false
40152024-02-06T22:17:15.436Z 11 Acked 1010 Write 9 Done Sent Done false
40162024-02-06T22:17:15.436Z 12 Acked 1011 Read 6 Done Sent Done false
40172024-02-06T22:17:15.436Z 13 Acked 1012 Write 3 Done Sent Done false
40182024-02-06T22:17:15.436Z 14 Acked 1013 Flush 0 Done Sent Done false
40192024-02-06T22:17:15.436Z 15 Acked 1014 Write 2 Done Sent Done false
40202024-02-06T22:17:15.436Z 16 Acked 1015 Read 6 Done Sent Done false
40212024-02-06T22:17:15.436Z 17 Acked 1016 Read 9 Done Sent Done false
40222024-02-06T22:17:15.436Z 18 Acked 1017 Read 9 Done Sent Done false
40232024-02-06T22:17:15.436Z 19 Acked 1018 Write 9 Done Sent Done false
40242024-02-06T22:17:15.436Z 20 Acked 1019 Write 9 Done Sent Done false
40252024-02-06T22:17:15.436Z 21 Acked 1020 Read 8 Done Sent Done false
40262024-02-06T22:17:15.436Z 22 Acked 1021 Flush 0 Done Sent Done false
40272024-02-06T22:17:15.436Z 23 Acked 1022 Flush 0 Done Sent Done false
40282024-02-06T22:17:15.436Z 24 Acked 1023 Write 6 Done Sent Done false
40292024-02-06T22:17:15.436Z 25 Acked 1024 Write 9 Done Sent Done false
40302024-02-06T22:17:15.436Z 26 Acked 1025 Write 9 Done Sent Done false
40312024-02-06T22:17:15.437Z 27 Acked 1026 Read 9 Done Sent Done false
40322024-02-06T22:17:15.437Z 28 Acked 1027 Write 9 Done Sent Done false
40332024-02-06T22:17:15.437Z 29 Acked 1028 Write 8 Done Sent Done false
40342024-02-06T22:17:15.437Z 30 Acked 1029 Read 4 Done Sent Sent false
40352024-02-06T22:17:15.437Z STATES DS:0 DS:1 DS:2 TOTAL
40362024-02-06T22:17:15.437Z New 0 0 0 0
40372024-02-06T22:17:15.437Z Sent 0 20 1 21
40382024-02-06T22:17:15.437Z Done 30 10 29 69
40392024-02-06T22:17:15.437Z Skipped 0 0 0 0
40402024-02-06T22:17:15.437Z Error 0 0 0 0
40412024-02-06T22:17:15.437ZLast Flush: 1022 0 1022
40422024-02-06T22:17:15.437ZDownstairs last five completed:
40432024-02-06T22:17:15.437ZUpstairs last five completed: 30 29 28 27 26
40442024-02-06T22:17:15.437Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:14.664546089Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1279}
40452024-02-06T22:17:15.437ZWrote out file "/tmp/test_repair_verify.data"
40462024-02-06T22:17:15.441Z
40472024-02-06T22:17:15.441ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40482024-02-06T22:17:15.458ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40492024-02-06T22:17:15.458Z 0 000-019 34 34 34 72 72 72 F F F
40502024-02-06T22:17:15.458Z 1 020-039 34 34 34 72 72 72 F F F
40512024-02-06T22:17:15.458Z 2 040-059 30 30 30 63 63 63 T F T <---
40522024-02-06T22:17:15.458Z 3 060-079 36 34 36 77 73 77 T T T <---
40532024-02-06T22:17:15.458Z 4 080-099 32 32 32 68 68 68 F F F
40542024-02-06T22:17:15.458Z 5 100-119 36 35 36 77 76 77 F T F <---
40552024-02-06T22:17:15.458Z 6 120-139 35 35 35 76 76 76 F F F
40562024-02-06T22:17:15.458Z 7 140-159 36 26 36 78 56 78 T T T <---
40572024-02-06T22:17:15.459Z 8 160-179 34 34 34 74 74 74 T F T <---
40582024-02-06T22:17:15.462Z 9 180-199 34 34 34 74 74 74 F F F
40592024-02-06T22:17:15.462Z 10 200-219 36 33 36 77 69 77 T T T <---
40602024-02-06T22:17:15.462Z 11 220-239 35 35 35 76 76 76 F F F
40612024-02-06T22:17:15.462Z 12 240-259 34 34 34 75 75 75 F F F
40622024-02-06T22:17:15.462Z 13 260-279 32 32 32 68 68 68 T F T <---
40632024-02-06T22:17:15.462Z 14 280-299 33 33 33 69 69 69 T F T <---
40642024-02-06T22:17:15.462Z 15 300-319 29 29 29 60 60 60 F F F
40652024-02-06T22:17:15.462Z 16 320-339 36 34 36 77 75 77 F F F <---
40662024-02-06T22:17:15.462Z 17 340-359 36 34 36 77 75 77 F T F <---
40672024-02-06T22:17:15.462Z 18 360-379 36 30 36 77 63 77 F T F <---
40682024-02-06T22:17:15.462Z 19 380-399 24 24 24 52 52 52 F F F
40692024-02-06T22:17:15.462Z 20 400-419 35 35 35 76 76 76 F F F
40702024-02-06T22:17:15.462Z 21 420-439 34 34 34 73 73 73 F F F
40712024-02-06T22:17:15.463Z 22 440-459 34 34 34 73 73 73 F F F
40722024-02-06T22:17:15.463Z 23 460-479 32 32 32 66 66 66 F F F
40732024-02-06T22:17:15.463Z 24 480-499 32 32 32 65 65 65 F F F
40742024-02-06T22:17:15.463Z 25 500-519 36 26 36 78 55 78 F F F <---
40752024-02-06T22:17:15.463Z 26 520-539 34 34 34 71 71 71 F F F
40762024-02-06T22:17:15.463Z 27 540-559 35 35 35 76 76 76 F F F
40772024-02-06T22:17:15.463Z 28 560-579 36 35 36 77 76 77 T F T <---
40782024-02-06T22:17:15.463Z 29 580-599 36 32 36 77 68 77 F T F <---
40792024-02-06T22:17:15.463ZMax gen: 36, Max flush: 78
40802024-02-06T22:17:15.463ZError: Difference in extent metadata found!
40812024-02-06T22:17:15.463ZA Difference in extent metadata is expected here
40822024-02-06T22:17:15.463ZOn loop 18
40832024-02-06T22:17:15.463Z
40842024-02-06T22:17:15.468ZVerifying data now
40852024-02-06T22:17:21.147ZNo change after verify
40862024-02-06T22:17:21.150ZLoop: 18 Downstairs dump after verify (and repair):
40872024-02-06T22:17:21.162ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40882024-02-06T22:17:21.162Z 0 000-019 34 34 34 72 72 72 F F F
40892024-02-06T22:17:21.162Z 1 020-039 34 34 34 72 72 72 F F F
40902024-02-06T22:17:21.162Z 2 040-059 37 37 37 79 79 79 F F F
40912024-02-06T22:17:21.162Z 3 060-079 37 37 37 79 79 79 F F F
40922024-02-06T22:17:21.162Z 4 080-099 32 32 32 68 68 68 F F F
40932024-02-06T22:17:21.162Z 5 100-119 36 36 36 77 77 77 F F F
40942024-02-06T22:17:21.162Z 6 120-139 35 35 35 76 76 76 F F F
40952024-02-06T22:17:21.163Z 7 140-159 37 37 37 79 79 79 F F F
40962024-02-06T22:17:21.163Z 8 160-179 37 37 37 79 79 79 F F F
40972024-02-06T22:17:21.166Z 9 180-199 34 34 34 74 74 74 F F F
40982024-02-06T22:17:21.166Z 10 200-219 37 37 37 79 79 79 F F F
40992024-02-06T22:17:21.166Z 11 220-239 35 35 35 76 76 76 F F F
41002024-02-06T22:17:21.166Z 12 240-259 34 34 34 75 75 75 F F F
41012024-02-06T22:17:21.166Z 13 260-279 37 37 37 79 79 79 F F F
41022024-02-06T22:17:21.166Z 14 280-299 37 37 37 79 79 79 F F F
41032024-02-06T22:17:21.166Z 15 300-319 29 29 29 60 60 60 F F F
41042024-02-06T22:17:21.166Z 16 320-339 36 36 36 77 77 77 F F F
41052024-02-06T22:17:21.166Z 17 340-359 36 36 36 77 77 77 F F F
41062024-02-06T22:17:21.166Z 18 360-379 36 36 36 77 77 77 F F F
41072024-02-06T22:17:21.166Z 19 380-399 24 24 24 52 52 52 F F F
41082024-02-06T22:17:21.166Z 20 400-419 35 35 35 76 76 76 F F F
41092024-02-06T22:17:21.166Z 21 420-439 34 34 34 73 73 73 F F F
41102024-02-06T22:17:21.166Z 22 440-459 34 34 34 73 73 73 F F F
41112024-02-06T22:17:21.166Z 23 460-479 32 32 32 66 66 66 F F F
41122024-02-06T22:17:21.167Z 24 480-499 32 32 32 65 65 65 F F F
41132024-02-06T22:17:21.167Z 25 500-519 36 36 36 78 78 78 F F F
41142024-02-06T22:17:21.167Z 26 520-539 34 34 34 71 71 71 F F F
41152024-02-06T22:17:21.167Z 27 540-559 35 35 35 76 76 76 F F F
41162024-02-06T22:17:21.167Z 28 560-579 37 37 37 79 79 79 F F F
41172024-02-06T22:17:21.167Z 29 580-599 36 36 36 77 77 77 F F F
41182024-02-06T22:17:21.167ZMax gen: 37, Max flush: 79
41192024-02-06T22:17:21.167Z
41202024-02-06T22:17:21.167ZBegin loop 19
41212024-02-06T22:17:21.167ZDownstairs to restart: 0
41222024-02-06T22:17:21.185Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.413788617Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287}
41232024-02-06T22:17:21.185Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414033733Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287}
41242024-02-06T22:17:21.190Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.41406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287}
41252024-02-06T22:17:21.190Z{"msg":"Crucible e842c97a-cff7-412c-95af-93cd292f63b1 has session id: 6194a251-ed23-49b9-af4e-1d7455c92865","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414075808Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41262024-02-06T22:17:21.191Z{"msg":"Upstairs opts: Upstairs UUID: e842c97a-cff7-412c-95af-93cd292f63b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
41272024-02-06T22:17:21.191Z,"time":"2024-02-06T22:17:20.414106284Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41282024-02-06T22:17:21.191Z{"msg":"Crucible stats registered with UUID: e842c97a-cff7-412c-95af-93cd292f63b1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414180964Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41292024-02-06T22:17:21.191Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414213799Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287}
41302024-02-06T22:17:21.191Z{"msg":"e842c97a-cff7-412c-95af-93cd292f63b1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414332303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41312024-02-06T22:17:21.191Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.41435844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41322024-02-06T22:17:21.191Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414379327Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41332024-02-06T22:17:21.191Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414403123Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41342024-02-06T22:17:21.191Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.41443013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41352024-02-06T22:17:21.191Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414469625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41362024-02-06T22:17:21.191Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414510309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41372024-02-06T22:17:21.191Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414539755Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41382024-02-06T22:17:21.191Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414594778Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41392024-02-06T22:17:21.191Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414637502Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41402024-02-06T22:17:21.191Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414664998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41412024-02-06T22:17:21.191Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414699643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41422024-02-06T22:17:21.191Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:20.414768424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41432024-02-06T22:17:21.191Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:20.414810819Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41442024-02-06T22:17:21.191Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:20.414838135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41452024-02-06T22:17:21.191Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414852163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41462024-02-06T22:17:21.191Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414881099Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41472024-02-06T22:17:21.191Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.414971327Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41482024-02-06T22:17:21.192Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.415373623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41492024-02-06T22:17:21.192Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.415408128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41502024-02-06T22:17:21.192Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.415434954Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41512024-02-06T22:17:21.192Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.415474379Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41522024-02-06T22:17:21.192Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.415974761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41532024-02-06T22:17:21.192Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.416016245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41542024-02-06T22:17:21.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.41657412Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41552024-02-06T22:17:21.192Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.416610905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41562024-02-06T22:17:21.192Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.416627223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41572024-02-06T22:17:21.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.41664758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41582024-02-06T22:17:21.192Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.416661558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41592024-02-06T22:17:21.192Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:20.416682066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41602024-02-06T22:17:22.186Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.41618781Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41612024-02-06T22:17:22.186Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.41633264Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41622024-02-06T22:17:22.192Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.41714913Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41632024-02-06T22:17:22.193Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.417184705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41642024-02-06T22:17:22.193Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.417632914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41652024-02-06T22:17:22.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418145015Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41662024-02-06T22:17:22.193Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418171792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41672024-02-06T22:17:22.193Z{"msg":"[0]R flush_numbers[0..12]: [72, 72, 79, 79, 68, 77, 76, 79, 79, 74, 79, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418186979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41682024-02-06T22:17:22.193Z{"msg":"[0]R generation[0..12]: [34, 34, 37, 37, 32, 36, 35, 37, 37, 34, 37, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418214456Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41692024-02-06T22:17:22.194Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418237133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41702024-02-06T22:17:22.194Z{"msg":"[1]R flush_numbers[0..12]: [72, 72, 79, 79, 68, 77, 76, 79, 79, 74, 79, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418261369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41712024-02-06T22:17:22.194Z{"msg":"[1]R generation[0..12]: [34, 34, 37, 37, 32, 36, 35, 37, 37, 34, 37, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418283806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41722024-02-06T22:17:22.194Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418298994Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41732024-02-06T22:17:22.194Z{"msg":"[2]R flush_numbers[0..12]: [72, 72, 79, 79, 68, 77, 76, 79, 79, 74, 79, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418313293Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41742024-02-06T22:17:22.194Z{"msg":"[2]R generation[0..12]: [34, 34, 37, 37, 32, 36, 35, 37, 37, 34, 37, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418327531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41752024-02-06T22:17:22.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418341939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41762024-02-06T22:17:22.194Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418355767Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41772024-02-06T22:17:22.195ZWait for a query_work_queue command to finish before sending IO
41782024-02-06T22:17:22.195Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418378794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41792024-02-06T22:17:22.195Z{"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418399021Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41802024-02-06T22:17:22.195Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418412599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41812024-02-06T22:17:22.195Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418427457Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41822024-02-06T22:17:22.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418440735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41832024-02-06T22:17:22.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418454453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41842024-02-06T22:17:22.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418468192Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41852024-02-06T22:17:22.195Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.41848168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41862024-02-06T22:17:22.196Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418494448Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41872024-02-06T22:17:22.196Z{"msg":"e842c97a-cff7-412c-95af-93cd292f63b1 is now active with session: 6194a251-ed23-49b9-af4e-1d7455c92865","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418508496Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41882024-02-06T22:17:22.196Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418520654Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287,"session_id":"6194a251-ed23-49b9-af4e-1d7455c92865"}
41892024-02-06T22:17:22.196Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:21.418533043Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287}
41902024-02-06T22:17:22.196ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41912024-02-06T22:17:22.196ZLoading write count information from file "/tmp/test_repair_verify.data"
41922024-02-06T22:17:22.197ZRun Repair workload
41932024-02-06T22:17:22.197Z01/30 Write block 156 len 12288 data: 5 6 6
41942024-02-06T22:17:22.197Z02/30 Flush
41952024-02-06T22:17:22.203Z03/30 Write block 509 len 24576 data: 3 3 3 2 5 4
41962024-02-06T22:17:22.203Z04/30 Write block 265 len 20480 data: 5 5 4 4 4
41972024-02-06T22:17:22.203Z05/30 Read block 113 len 16384
41982024-02-06T22:17:22.203Z06/30 Read block 413 len 4096
41992024-02-06T22:17:22.203Z07/30 Read block 66 len 36864
42002024-02-06T22:17:22.203Z08/30 Read block 90 len 4096
42012024-02-06T22:17:22.203Z09/30 Write block 14 len 16384 data: 5 6 5 4
42022024-02-06T22:17:22.204Z10/30 Read block 199 len 36864
42032024-02-06T22:17:22.204Z11/30 Read block 44 len 20480
42042024-02-06T22:17:22.204Z12/30 Read block 550 len 36864
42052024-02-06T22:17:22.204Z13/30 Read block 208 len 24576
42062024-02-06T22:17:22.204Z14/30 Write block 372 len 16384 data: 2 2 4 4
42072024-02-06T22:17:22.230Z15/30 Write block 37 len 40960 data: 2 2 2 3 2 2 2 2 2 3
42082024-02-06T22:17:22.230Z16/30 Write block 50 len 28672 data: 7 9 9 10 10 10 7
42092024-02-06T22:17:22.231Z17/30 Read block 129 len 8192
42102024-02-06T22:17:22.231Z18/30 Read block 58 len 32768
42112024-02-06T22:17:22.231Z19/30 Write block 479 len 4096 data: 5
42122024-02-06T22:17:22.231Z20/30 Read block 267 len 12288
42132024-02-06T22:17:22.231Z21/30 Read block 545 len 28672
42142024-02-06T22:17:22.231Z22/30 Write block 390 len 8192 data: 3 4
42152024-02-06T22:17:22.231Z23/30 Read block 405 len 20480
42162024-02-06T22:17:22.232Z24/30 Write block 36 len 36864 data: 2 3 3 3 4 3 3 3 3
42172024-02-06T22:17:22.232Z25/30 Write block 419 len 32768 data: 5 4 4 3 3 4 4 5
42182024-02-06T22:17:22.232Z26/30 Read block 538 len 24576
42192024-02-06T22:17:22.232Z27/30 Write block 414 len 32768 data: 3 4 4 6 6 6 5 5
42202024-02-06T22:17:22.232Z28/30 Write block 194 len 28672 data: 3 3 3 4 3 3 3
42212024-02-06T22:17:22.232Z29/30 Write block 172 len 16384 data: 4 4 3 2
42222024-02-06T22:17:22.233Z30/30 Write block 22 len 16384 data: 5 5 4 5
42232024-02-06T22:17:22.233Z----------------------------------------------------------------
42242024-02-06T22:17:22.233Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
42252024-02-06T22:17:22.233ZStates: Active Active Active
42262024-02-06T22:17:22.233ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42272024-02-06T22:17:22.233Z 1 Acked 1000 Write 3 Done Done Done false
42282024-02-06T22:17:22.233Z 2 Acked 1001 Flush 0 Sent Done Done false
42292024-02-06T22:17:22.234Z 3 Acked 1002 Write 6 Sent Done Done false
42302024-02-06T22:17:22.234Z 4 Acked 1003 Write 5 Sent Done Done false
42312024-02-06T22:17:22.234Z 5 Acked 1004 Read 4 Sent Done Done false
42322024-02-06T22:17:22.234Z 6 Acked 1005 Read 1 Sent Done Done false
42332024-02-06T22:17:22.234Z 7 Acked 1006 Read 9 Sent Done Done false
42342024-02-06T22:17:22.234Z 8 Acked 1007 Read 1 Sent Done Done false
42352024-02-06T22:17:22.234Z 9 Acked 1008 Write 4 Sent Done Done false
42362024-02-06T22:17:22.235Z 10 Acked 1009 Read 9 Sent Done Done false
42372024-02-06T22:17:22.235Z 11 Acked 1010 Read 5 Sent Done Done false
42382024-02-06T22:17:22.235Z 12 Acked 1011 Read 9 Sent Done Done false
42392024-02-06T22:17:22.235Z 13 Acked 1012 Read 6 Sent Done Done false
42402024-02-06T22:17:22.235Z 14 Acked 1013 Write 4 Sent Done Done false
42412024-02-06T22:17:22.235Z 15 Acked 1014 Write 10 Sent Done Done false
42422024-02-06T22:17:22.236Z 16 Acked 1015 Write 7 Sent Done Done false
42432024-02-06T22:17:22.236Z 17 Acked 1016 Read 2 Sent Done Done false
42442024-02-06T22:17:22.236Z 18 Acked 1017 Read 8 Sent Done Done false
42452024-02-06T22:17:22.236Z 19 Acked 1018 Write 1 Sent Done Done false
42462024-02-06T22:17:22.236Z 20 Acked 1019 Read 3 Sent Done Done false
42472024-02-06T22:17:22.236Z 21 Acked 1020 Read 7 Sent Done Done false
42482024-02-06T22:17:22.236Z 22 Acked 1021 Write 2 Sent Done Done false
42492024-02-06T22:17:22.237Z 23 Acked 1022 Read 5 Sent Done Done false
42502024-02-06T22:17:22.237Z 24 Acked 1023 Write 9 Sent Done Done false
42512024-02-06T22:17:22.237Z 25 Acked 1024 Write 8 Sent Done Done false
42522024-02-06T22:17:22.237Z 26 Acked 1025 Read 6 Sent Done Done false
42532024-02-06T22:17:22.237Z 27 Acked 1026 Write 8 Sent Done Done false
42542024-02-06T22:17:22.237Z 28 Acked 1027 Write 7 Sent Done Done false
42552024-02-06T22:17:22.238Z 29 Acked 1028 Write 4 Sent Done Done false
42562024-02-06T22:17:22.238Z 30 Acked 1029 Write 4 Sent Sent Sent false
42572024-02-06T22:17:22.238Z STATES DS:0 DS:1 DS:2 TOTAL
42582024-02-06T22:17:22.238Z New 0 0 0 0
42592024-02-06T22:17:22.238Z Sent 29 1 1 31
42602024-02-06T22:17:22.238Z Done 1 29 29 59
42612024-02-06T22:17:22.238Z Skipped 0 0 0 0
42622024-02-06T22:17:22.239Z Error 0 0 0 0
42632024-02-06T22:17:22.239ZLast Flush: 0 1001 1001
42642024-02-06T22:17:22.239ZDownstairs last five completed:
42652024-02-06T22:17:22.239ZUpstairs last five completed: 30 29 28 27 26
42662024-02-06T22:17:22.239Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:21.44697175Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1287}
42672024-02-06T22:17:22.239ZWrote out file "/tmp/test_repair_verify.data"
42682024-02-06T22:17:22.239Z
42692024-02-06T22:17:22.239ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42702024-02-06T22:17:22.253ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42712024-02-06T22:17:22.254Z 0 000-019 34 34 34 72 72 72 F T T <---
42722024-02-06T22:17:22.254Z 1 020-039 34 34 34 72 72 72 F T T <---
42732024-02-06T22:17:22.254Z 2 040-059 37 37 37 79 79 79 F T T <---
42742024-02-06T22:17:22.254Z 3 060-079 37 37 37 79 79 79 F F F
42752024-02-06T22:17:22.254Z 4 080-099 32 32 32 68 68 68 F F F
42762024-02-06T22:17:22.254Z 5 100-119 36 36 36 77 77 77 F F F
42772024-02-06T22:17:22.254Z 6 120-139 35 35 35 76 76 76 F F F
42782024-02-06T22:17:22.254Z 7 140-159 37 38 38 79 80 80 T F F <---
42792024-02-06T22:17:22.254Z 8 160-179 37 37 37 79 79 79 F T T <---
42802024-02-06T22:17:22.255Z 9 180-199 34 34 34 74 74 74 F T T <---
42812024-02-06T22:17:22.255Z 10 200-219 37 37 37 79 79 79 F T T <---
42822024-02-06T22:17:22.255Z 11 220-239 35 35 35 76 76 76 F F F
42832024-02-06T22:17:22.255Z 12 240-259 34 34 34 75 75 75 F F F
42842024-02-06T22:17:22.255Z 13 260-279 37 37 37 79 79 79 F T T <---
42852024-02-06T22:17:22.255Z 14 280-299 37 37 37 79 79 79 F F F
42862024-02-06T22:17:22.256Z 15 300-319 29 29 29 60 60 60 F F F
42872024-02-06T22:17:22.256Z 16 320-339 36 36 36 77 77 77 F F F
42882024-02-06T22:17:22.256Z 17 340-359 36 36 36 77 77 77 F F F
42892024-02-06T22:17:22.256Z 18 360-379 36 36 36 77 77 77 F T T <---
42902024-02-06T22:17:22.256Z 19 380-399 24 24 24 52 52 52 F T T <---
42912024-02-06T22:17:22.256Z 20 400-419 35 35 35 76 76 76 F T T <---
42922024-02-06T22:17:22.256Z 21 420-439 34 34 34 73 73 73 F T T <---
42932024-02-06T22:17:22.256Z 22 440-459 34 34 34 73 73 73 F F F
42942024-02-06T22:17:22.256Z 23 460-479 32 32 32 66 66 66 F T T <---
42952024-02-06T22:17:22.257Z 24 480-499 32 32 32 65 65 65 F F F
42962024-02-06T22:17:22.257Z 25 500-519 36 36 36 78 78 78 F T T <---
42972024-02-06T22:17:22.257Z 26 520-539 34 34 34 71 71 71 F F F
42982024-02-06T22:17:22.257Z 27 540-559 35 35 35 76 76 76 F F F
42992024-02-06T22:17:22.257Z 28 560-579 37 37 37 79 79 79 F F F
43002024-02-06T22:17:22.257Z 29 580-599 36 36 36 77 77 77 F F F
43012024-02-06T22:17:22.258ZMax gen: 38, Max flush: 80
43022024-02-06T22:17:22.258ZError: Difference in extent metadata found!
43032024-02-06T22:17:22.263ZA Difference in extent metadata is expected here
43042024-02-06T22:17:22.264ZOn loop 19
43052024-02-06T22:17:22.264Z
43062024-02-06T22:17:22.264ZVerifying data now
43072024-02-06T22:17:26.950ZNo change after verify
43082024-02-06T22:17:26.950ZLoop: 19 Downstairs dump after verify (and repair):
43092024-02-06T22:17:26.966ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43102024-02-06T22:17:26.966Z 0 000-019 39 39 39 81 81 81 F F F
43112024-02-06T22:17:26.966Z 1 020-039 39 39 39 81 81 81 F F F
43122024-02-06T22:17:26.966Z 2 040-059 39 39 39 81 81 81 F F F
43132024-02-06T22:17:26.966Z 3 060-079 37 37 37 79 79 79 F F F
43142024-02-06T22:17:26.967Z 4 080-099 32 32 32 68 68 68 F F F
43152024-02-06T22:17:26.967Z 5 100-119 36 36 36 77 77 77 F F F
43162024-02-06T22:17:26.967Z 6 120-139 35 35 35 76 76 76 F F F
43172024-02-06T22:17:26.967Z 7 140-159 38 38 38 80 80 80 F F F
43182024-02-06T22:17:26.967Z 8 160-179 39 39 39 81 81 81 F F F
43192024-02-06T22:17:26.967Z 9 180-199 39 39 39 81 81 81 F F F
43202024-02-06T22:17:26.971Z 10 200-219 39 39 39 81 81 81 F F F
43212024-02-06T22:17:26.971Z 11 220-239 35 35 35 76 76 76 F F F
43222024-02-06T22:17:26.971Z 12 240-259 34 34 34 75 75 75 F F F
43232024-02-06T22:17:26.971Z 13 260-279 39 39 39 81 81 81 F F F
43242024-02-06T22:17:26.971Z 14 280-299 37 37 37 79 79 79 F F F
43252024-02-06T22:17:26.971Z 15 300-319 29 29 29 60 60 60 F F F
43262024-02-06T22:17:26.971Z 16 320-339 36 36 36 77 77 77 F F F
43272024-02-06T22:17:26.971Z 17 340-359 36 36 36 77 77 77 F F F
43282024-02-06T22:17:26.971Z 18 360-379 39 39 39 81 81 81 F F F
43292024-02-06T22:17:26.972Z 19 380-399 39 39 39 81 81 81 F F F
43302024-02-06T22:17:26.972Z 20 400-419 39 39 39 81 81 81 F F F
43312024-02-06T22:17:26.972Z 21 420-439 39 39 39 81 81 81 F F F
43322024-02-06T22:17:26.972Z 22 440-459 34 34 34 73 73 73 F F F
43332024-02-06T22:17:26.972Z 23 460-479 39 39 39 81 81 81 F F F
43342024-02-06T22:17:26.972Z 24 480-499 32 32 32 65 65 65 F F F
43352024-02-06T22:17:26.972Z 25 500-519 39 39 39 81 81 81 F F F
43362024-02-06T22:17:26.972Z 26 520-539 34 34 34 71 71 71 F F F
43372024-02-06T22:17:26.972Z 27 540-559 35 35 35 76 76 76 F F F
43382024-02-06T22:17:26.972Z 28 560-579 37 37 37 79 79 79 F F F
43392024-02-06T22:17:26.972Z 29 580-599 36 36 36 77 77 77 F F F
43402024-02-06T22:17:26.972ZMax gen: 39, Max flush: 81
43412024-02-06T22:17:26.972Z
43422024-02-06T22:17:26.973ZBegin loop 20
43432024-02-06T22:17:26.973ZDownstairs to restart: 1
43442024-02-06T22:17:26.989Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218419868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295}
43452024-02-06T22:17:26.995Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.21870808Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295}
43462024-02-06T22:17:26.996Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218740655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295}
43472024-02-06T22:17:26.996ZCrucible runtime is spawned
43482024-02-06T22:17:26.996Z{"msg":"Crucible fc51525d-90b8-446b-8f95-8c720dbb47e8 has session id: dd001850-36de-48eb-802b-0caf0ffacd1c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218783089Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43492024-02-06T22:17:26.996Z{"msg":"Upstairs opts: Upstairs UUID: fc51525d-90b8-446b-8f95-8c720dbb47e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218822304Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43502024-02-06T22:17:26.996Z{"msg":"Crucible stats registered with UUID: fc51525d-90b8-446b-8f95-8c720dbb47e8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218845691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43512024-02-06T22:17:26.996Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218867098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295}
43522024-02-06T22:17:26.996Z{"msg":"fc51525d-90b8-446b-8f95-8c720dbb47e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218947777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43532024-02-06T22:17:26.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.218987582Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43542024-02-06T22:17:26.996Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219016498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43552024-02-06T22:17:26.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219052983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43562024-02-06T22:17:26.996Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219100527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43572024-02-06T22:17:26.996Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219138811Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43582024-02-06T22:17:26.996Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219166408Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43592024-02-06T22:17:26.996Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219193054Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43602024-02-06T22:17:26.996Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219226819Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43612024-02-06T22:17:26.996Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219262355Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43622024-02-06T22:17:26.996Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.21929435Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43632024-02-06T22:17:26.996Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219312838Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43642024-02-06T22:17:26.996Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:26.219363981Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43652024-02-06T22:17:26.996Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:26.219393417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43662024-02-06T22:17:26.997Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:26.219434222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43672024-02-06T22:17:26.997Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219459408Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43682024-02-06T22:17:26.997Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219491104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43692024-02-06T22:17:26.997Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.219651372Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43702024-02-06T22:17:26.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.220128568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43712024-02-06T22:17:26.997Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.220165563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43722024-02-06T22:17:26.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.2201829Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43732024-02-06T22:17:26.997Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.220205707Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43742024-02-06T22:17:26.997Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.220740305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43752024-02-06T22:17:26.997Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.220877196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43762024-02-06T22:17:26.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.221244267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43772024-02-06T22:17:26.997Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.22129708Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43782024-02-06T22:17:26.997Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.221333875Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43792024-02-06T22:17:26.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.221358051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43802024-02-06T22:17:26.997Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.221386588Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43812024-02-06T22:17:26.997Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:26.221410904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43822024-02-06T22:17:27.991Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.220875834Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43832024-02-06T22:17:27.991Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.220995278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43842024-02-06T22:17:27.996Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.22209072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43852024-02-06T22:17:27.996Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.222132594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43862024-02-06T22:17:27.996Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.222578234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43872024-02-06T22:17:27.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223029863Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43882024-02-06T22:17:27.996Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223078856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43892024-02-06T22:17:27.996Z{"msg":"[0]R flush_numbers[0..12]: [81, 81, 81, 79, 68, 77, 76, 80, 81, 81, 81, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223099453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43902024-02-06T22:17:27.996Z{"msg":"[0]R generation[0..12]: [39, 39, 39, 37, 32, 36, 35, 38, 39, 39, 39, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.22312458Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43912024-02-06T22:17:27.996Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223141158Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43922024-02-06T22:17:27.996Z{"msg":"[1]R flush_numbers[0..12]: [81, 81, 81, 79, 68, 77, 76, 80, 81, 81, 81, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223162915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43932024-02-06T22:17:27.996Z{"msg":"[1]R generation[0..12]: [39, 39, 39, 37, 32, 36, 35, 38, 39, 39, 39, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223190811Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43942024-02-06T22:17:27.996Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
43952024-02-06T22:17:27.996Z,"time":"2024-02-06T22:17:27.223213798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43962024-02-06T22:17:27.997Z{"msg":"[2]R flush_numbers[0..12]: [81, 81, 81, 79, 68, 77, 76, 80, 81, 81, 81, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223228356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43972024-02-06T22:17:27.997Z{"msg":"[2]R generation[0..12]: [39, 39, 39, 37, 32, 36, 35, 38, 39, 39, 39, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223242214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43982024-02-06T22:17:27.997Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223263941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
43992024-02-06T22:17:27.997Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223279159Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44002024-02-06T22:17:27.997Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223292857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44012024-02-06T22:17:27.997Z{"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223314394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44022024-02-06T22:17:27.997Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223328223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44032024-02-06T22:17:27.997Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.22334877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44042024-02-06T22:17:27.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223361908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44052024-02-06T22:17:27.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223382105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44062024-02-06T22:17:27.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223395773Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44072024-02-06T22:17:27.997Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223410172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44082024-02-06T22:17:27.997Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223423Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44092024-02-06T22:17:27.997Z{"msg":"fc51525d-90b8-446b-8f95-8c720dbb47e8 is now active with session: dd001850-36de-48eb-802b-0caf0ffacd1c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223443717Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44102024-02-06T22:17:27.997Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223457135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295,"session_id":"dd001850-36de-48eb-802b-0caf0ffacd1c"}
44112024-02-06T22:17:27.997Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:27.223470053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295}
44122024-02-06T22:17:27.997ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
44132024-02-06T22:17:27.997ZLoading write count information from file "/tmp/test_repair_verify.data"
44142024-02-06T22:17:27.997ZRun Repair workload
44152024-02-06T22:17:27.997Z01/30 Write block 268 len 36864 data: 5 5 4 3 3 2 2 2 3
44162024-02-06T22:17:28.001Z02/30 Read block 251 len 24576
44172024-02-06T22:17:28.001Z03/30 Read block 444 len 16384
44182024-02-06T22:17:28.001Z04/30 Write block 378 len 12288 data: 7 6 6
44192024-02-06T22:17:28.001Z05/30 Write block 368 len 40960 data: 5 5 4 4 3 3 5 5 6 8
44202024-02-06T22:17:28.001Z06/30 Write block 436 len 8192 data: 5 5
44212024-02-06T22:17:28.002Z07/30 Write block 210 len 16384 data: 6 4 4 4
44222024-02-06T22:17:28.002Z08/30 Write block 562 len 20480 data: 3 3 4 4 3
44232024-02-06T22:17:28.005Z09/30 Read block 282 len 28672
44242024-02-06T22:17:28.005Z10/30 Write block 154 len 16384 data: 7 7 6 7
44252024-02-06T22:17:28.005Z11/30 Read block 519 len 16384
44262024-02-06T22:17:28.005Z12/30 Write block 544 len 4096 data: 5
44272024-02-06T22:17:28.005Z13/30 Write block 254 len 4096 data: 5
44282024-02-06T22:17:28.005Z14/30 Read block 187 len 28672
44292024-02-06T22:17:28.008Z15/30 Read block 117 len 12288
44302024-02-06T22:17:28.008Z16/30 Read block 151 len 20480
44312024-02-06T22:17:28.008Z17/30 Read block 326 len 24576
44322024-02-06T22:17:28.008Z18/30 Read block 16 len 16384
44332024-02-06T22:17:28.011Z19/30 Write block 86 len 16384 data: 3 3 3 3
44342024-02-06T22:17:28.011Z20/30 Flush
44352024-02-06T22:17:28.014Z21/30 Write block 254 len 40960 data: 6 4 3 3 2 3 4 5 5 5
44362024-02-06T22:17:28.014Z22/30 Read block 298 len 40960
44372024-02-06T22:17:28.017Z23/30 Write block 477 len 8192 data: 6 6
44382024-02-06T22:17:28.017Z24/30 Write block 589 len 40960 data: 5 7 7 6 6 5 6 5 5 5
44392024-02-06T22:17:28.017Z25/30 Read block 484 len 36864
44402024-02-06T22:17:28.020Z26/30 Write block 173 len 24576 data: 5 4 3 3 4 5
44412024-02-06T22:17:28.020Z27/30 Write block 502 len 24576 data: 5 6 4 4 5 5
44422024-02-06T22:17:28.020Z28/30 Write block 142 len 4096 data: 4
44432024-02-06T22:17:28.020Z29/30 Read block 373 len 40960
44442024-02-06T22:17:28.024Z30/30 Write block 184 len 28672 data: 8 8 6 6 5 3 2
44452024-02-06T22:17:28.024Z----------------------------------------------------------------
44462024-02-06T22:17:28.024Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44472024-02-06T22:17:28.024ZStates: Active Active Active
44482024-02-06T22:17:28.024ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44492024-02-06T22:17:28.024Z 1 Acked 1000 Write 9 Done Done Done false
44502024-02-06T22:17:28.024Z 2 Acked 1001 Read 6 Done Done Done false
44512024-02-06T22:17:28.024Z 3 Acked 1002 Read 4 Done Sent Done false
44522024-02-06T22:17:28.024Z 4 Acked 1003 Write 3 Done Sent Done false
44532024-02-06T22:17:28.024Z 5 Acked 1004 Write 10 Done Sent Done false
44542024-02-06T22:17:28.024Z 6 Acked 1005 Write 2 Done Sent Done false
44552024-02-06T22:17:28.024Z 7 Acked 1006 Write 4 Done Sent Done false
44562024-02-06T22:17:28.024Z 8 Acked 1007 Write 5 Done Sent Done false
44572024-02-06T22:17:28.024Z 9 Acked 1008 Read 7 Done Sent Done false
44582024-02-06T22:17:28.025Z 10 Acked 1009 Write 4 Done Sent Done false
44592024-02-06T22:17:28.025Z 11 Acked 1010 Read 4 Done Sent Done false
44602024-02-06T22:17:28.025Z 12 Acked 1011 Write 1 Done Sent Done false
44612024-02-06T22:17:28.025Z 13 Acked 1012 Write 1 Done Sent Done false
44622024-02-06T22:17:28.025Z 14 Acked 1013 Read 7 Done Sent Done false
44632024-02-06T22:17:28.025Z 15 Acked 1014 Read 3 Done Sent Done false
44642024-02-06T22:17:28.025Z 16 Acked 1015 Read 5 Done Sent Done false
44652024-02-06T22:17:28.025Z 17 Acked 1016 Read 6 Done Sent Done false
44662024-02-06T22:17:28.025Z 18 Acked 1017 Read 4 Done Sent Done false
44672024-02-06T22:17:28.025Z 19 Acked 1018 Write 4 Done Sent Done false
44682024-02-06T22:17:28.025Z 20 Acked 1019 Flush 0 Done Sent Done false
44692024-02-06T22:17:28.025Z 21 Acked 1020 Write 10 Done Sent Done false
44702024-02-06T22:17:28.025Z 22 Acked 1021 Read 10 Done Sent Done false
44712024-02-06T22:17:28.025Z 23 Acked 1022 Write 2 Done Sent Done false
44722024-02-06T22:17:28.025Z 24 Acked 1023 Write 10 Done Sent Done false
44732024-02-06T22:17:28.025Z 25 Acked 1024 Read 9 Done Sent Done false
44742024-02-06T22:17:28.025Z 26 Acked 1025 Write 6 Done Sent Done false
44752024-02-06T22:17:28.025Z 27 Acked 1026 Write 6 Done Sent Done false
44762024-02-06T22:17:28.025Z 28 Acked 1027 Write 1 Done Sent Done false
44772024-02-06T22:17:28.025Z 29 Acked 1028 Read 10 Done Sent Done false
44782024-02-06T22:17:28.025Z 30 Acked 1029 Write 7 Sent Sent Sent false
44792024-02-06T22:17:28.025Z STATES DS:0 DS:1 DS:2 TOTAL
44802024-02-06T22:17:28.025Z New 0 0 0 0
44812024-02-06T22:17:28.026Z Sent 1 28 1 30
44822024-02-06T22:17:28.026Z Done 29 2 29 60
44832024-02-06T22:17:28.026Z Skipped 0 0 0 0
44842024-02-06T22:17:28.026Z Error 0 0 0 0
44852024-02-06T22:17:28.026ZLast Flush: 1019 0 1019
44862024-02-06T22:17:28.026ZDownstairs last five completed:
44872024-02-06T22:17:28.026ZUpstairs last five completed: 30 29 28 27 26
44882024-02-06T22:17:28.026Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:27.253296983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1295}
44892024-02-06T22:17:28.026ZWrote out file "/tmp/test_repair_verify.data"
44902024-02-06T22:17:28.030Z
44912024-02-06T22:17:28.030ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44922024-02-06T22:17:28.049ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44932024-02-06T22:17:28.049Z 0 000-019 39 39 39 81 81 81 F F F
44942024-02-06T22:17:28.049Z 1 020-039 39 39 39 81 81 81 F F F
44952024-02-06T22:17:28.049Z 2 040-059 39 39 39 81 81 81 F F F
44962024-02-06T22:17:28.049Z 3 060-079 37 37 37 79 79 79 F F F
44972024-02-06T22:17:28.049Z 4 080-099 40 32 40 82 68 82 F F F <---
44982024-02-06T22:17:28.049Z 5 100-119 36 36 36 77 77 77 F F F
44992024-02-06T22:17:28.049Z 6 120-139 35 35 35 76 76 76 F F F
45002024-02-06T22:17:28.050Z 7 140-159 40 38 40 82 80 82 T F T <---
45012024-02-06T22:17:28.050Z 8 160-179 39 39 39 81 81 81 T F T <---
45022024-02-06T22:17:28.050Z 9 180-199 39 39 39 81 81 81 T F T <---
45032024-02-06T22:17:28.050Z 10 200-219 40 39 40 82 81 82 F F F <---
45042024-02-06T22:17:28.050Z 11 220-239 35 35 35 76 76 76 F F F
45052024-02-06T22:17:28.050Z 12 240-259 40 34 40 82 75 82 T F T <---
45062024-02-06T22:17:28.050Z 13 260-279 40 39 40 82 81 82 T T T <---
45072024-02-06T22:17:28.053Z 14 280-299 37 37 37 79 79 79 F F F
45082024-02-06T22:17:28.053Z 15 300-319 29 29 29 60 60 60 F F F
45092024-02-06T22:17:28.053Z 16 320-339 36 36 36 77 77 77 F F F
45102024-02-06T22:17:28.053Z 17 340-359 36 36 36 77 77 77 F F F
45112024-02-06T22:17:28.053Z 18 360-379 40 39 40 82 81 82 F F F <---
45122024-02-06T22:17:28.053Z 19 380-399 40 39 40 82 81 82 F F F <---
45132024-02-06T22:17:28.053Z 20 400-419 39 39 39 81 81 81 F F F
45142024-02-06T22:17:28.053Z 21 420-439 40 39 40 82 81 82 F F F <---
45152024-02-06T22:17:28.053Z 22 440-459 34 34 34 73 73 73 F F F
45162024-02-06T22:17:28.053Z 23 460-479 39 39 39 81 81 81 T F T <---
45172024-02-06T22:17:28.053Z 24 480-499 32 32 32 65 65 65 F F F
45182024-02-06T22:17:28.053Z 25 500-519 39 39 39 81 81 81 T F T <---
45192024-02-06T22:17:28.054Z 26 520-539 34 34 34 71 71 71 F F F
45202024-02-06T22:17:28.054Z 27 540-559 40 35 40 82 76 82 F F F <---
45212024-02-06T22:17:28.054Z 28 560-579 40 37 40 82 79 82 F F F <---
45222024-02-06T22:17:28.054Z 29 580-599 36 36 36 77 77 77 T F T <---
45232024-02-06T22:17:28.054ZMax gen: 40, Max flush: 82
45242024-02-06T22:17:28.054ZError: Difference in extent metadata found!
45252024-02-06T22:17:28.054ZA Difference in extent metadata is expected here
45262024-02-06T22:17:28.054ZOn loop 20
45272024-02-06T22:17:28.054Z
45282024-02-06T22:17:28.057ZVerifying data now
45292024-02-06T22:17:32.777ZNo change after verify
45302024-02-06T22:17:32.777ZLoop: 20 Downstairs dump after verify (and repair):
45312024-02-06T22:17:32.791ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45322024-02-06T22:17:32.792Z 0 000-019 39 39 39 81 81 81 F F F
45332024-02-06T22:17:32.792Z 1 020-039 39 39 39 81 81 81 F F F
45342024-02-06T22:17:32.792Z 2 040-059 39 39 39 81 81 81 F F F
45352024-02-06T22:17:32.792Z 3 060-079 37 37 37 79 79 79 F F F
45362024-02-06T22:17:32.792Z 4 080-099 40 40 40 82 82 82 F F F
45372024-02-06T22:17:32.792Z 5 100-119 36 36 36 77 77 77 F F F
45382024-02-06T22:17:32.795Z 6 120-139 35 35 35 76 76 76 F F F
45392024-02-06T22:17:32.795Z 7 140-159 41 41 41 83 83 83 F F F
45402024-02-06T22:17:32.795Z 8 160-179 41 41 41 83 83 83 F F F
45412024-02-06T22:17:32.795Z 9 180-199 41 41 41 83 83 83 F F F
45422024-02-06T22:17:32.795Z 10 200-219 40 40 40 82 82 82 F F F
45432024-02-06T22:17:32.795Z 11 220-239 35 35 35 76 76 76 F F F
45442024-02-06T22:17:32.795Z 12 240-259 41 41 41 83 83 83 F F F
45452024-02-06T22:17:32.795Z 13 260-279 41 41 41 83 83 83 F F F
45462024-02-06T22:17:32.795Z 14 280-299 37 37 37 79 79 79 F F F
45472024-02-06T22:17:32.795Z 15 300-319 29 29 29 60 60 60 F F F
45482024-02-06T22:17:32.795Z 16 320-339 36 36 36 77 77 77 F F F
45492024-02-06T22:17:32.795Z 17 340-359 36 36 36 77 77 77 F F F
45502024-02-06T22:17:32.795Z 18 360-379 40 40 40 82 82 82 F F F
45512024-02-06T22:17:32.796Z 19 380-399 40 40 40 82 82 82 F F F
45522024-02-06T22:17:32.796Z 20 400-419 39 39 39 81 81 81 F F F
45532024-02-06T22:17:32.796Z 21 420-439 40 40 40 82 82 82 F F F
45542024-02-06T22:17:32.796Z 22 440-459 34 34 34 73 73 73 F F F
45552024-02-06T22:17:32.796Z 23 460-479 41 41 41 83 83 83 F F F
45562024-02-06T22:17:32.796Z 24 480-499 32 32 32 65 65 65 F F F
45572024-02-06T22:17:32.796Z 25 500-519 41 41 41 83 83 83 F F F
45582024-02-06T22:17:32.796Z 26 520-539 34 34 34 71 71 71 F F F
45592024-02-06T22:17:32.796Z 27 540-559 40 40 40 82 82 82 F F F
45602024-02-06T22:17:32.796Z 28 560-579 40 40 40 82 82 82 F F F
45612024-02-06T22:17:32.796Z 29 580-599 41 41 41 83 83 83 F F F
45622024-02-06T22:17:32.796ZMax gen: 41, Max flush: 83
45632024-02-06T22:17:32.796Z
45642024-02-06T22:17:32.796ZBegin loop 21
45652024-02-06T22:17:32.796ZDownstairs to restart: 2
45662024-02-06T22:17:32.932Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041081175Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303}
45672024-02-06T22:17:32.932Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041317513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303}
45682024-02-06T22:17:32.941Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041345269Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303}
45692024-02-06T22:17:32.941Z{"msg":"Crucible 12de417d-56a8-4c6e-a751-5d3ca9049c61 has session id: 5474e5dd-8914-4f9a-b7dc-c79cd79d0863","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041380684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45702024-02-06T22:17:32.941Z{"msg":"Upstairs opts: Upstairs UUID: 12de417d-56a8-4c6e-a751-5d3ca9049c61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041429068Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal"Crucible runtime is spawned
45712024-02-06T22:17:32.941Z,"pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45722024-02-06T22:17:32.941Z{"msg":"Crucible stats registered with UUID: 12de417d-56a8-4c6e-a751-5d3ca9049c61","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041528614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45732024-02-06T22:17:32.941Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041567029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303}
45742024-02-06T22:17:32.941Z{"msg":"12de417d-56a8-4c6e-a751-5d3ca9049c61 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041589716Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45752024-02-06T22:17:32.941Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041617102Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45762024-02-06T22:17:32.941Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041660117Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45772024-02-06T22:17:32.942Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041685843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45782024-02-06T22:17:32.942Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.04171168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45792024-02-06T22:17:32.942Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041757943Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45802024-02-06T22:17:32.942Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.04178401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45812024-02-06T22:17:32.942Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041816966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45822024-02-06T22:17:32.942Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041866659Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45832024-02-06T22:17:32.942Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041905443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45842024-02-06T22:17:32.942Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041940679Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45852024-02-06T22:17:32.942Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.041976534Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45862024-02-06T22:17:32.942Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.042034626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45872024-02-06T22:17:32.942Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.042735031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45882024-02-06T22:17:32.942Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.042773176Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45892024-02-06T22:17:32.942Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.042790104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45902024-02-06T22:17:32.942Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.042822839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45912024-02-06T22:17:32.942Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.043435107Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45922024-02-06T22:17:32.942Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.043476671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45932024-02-06T22:17:32.942Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.043600625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45942024-02-06T22:17:32.942Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.043638199Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45952024-02-06T22:17:32.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044142041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45962024-02-06T22:17:32.942Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044191685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45972024-02-06T22:17:32.942Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044217921Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45982024-02-06T22:17:32.942Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044245187Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
45992024-02-06T22:17:32.942Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044270914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46002024-02-06T22:17:32.942Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044304079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46012024-02-06T22:17:32.943Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044328436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46022024-02-06T22:17:32.943Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044712474Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46032024-02-06T22:17:32.943Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044770996Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46042024-02-06T22:17:32.943Z{"msg":"[0]R flush_numbers[0..12]: [81, 81, 81, 79, 82, 77, 76, 83, 83, 83, 82, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044807951Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46052024-02-06T22:17:32.943Z{"msg":"[0]R generation[0..12]: [39, 39, 39, 37, 40, 36, 35, 41, 41, 41, 40, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044825579Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46062024-02-06T22:17:32.943Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044849186Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46072024-02-06T22:17:32.943Z{"msg":"[1]R flush_numbers[0..12]: [81, 81, 81, 79, 82, 77, 76, 83, 83, 83, 82, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044864274Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46082024-02-06T22:17:32.943Z{"msg":"[1]R generation[0..12]: [39, 39, 39, 37, 40, 36, 35, 41, 41, 41, 40, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044885501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46092024-02-06T22:17:32.943Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","Wait for a query_work_queue command to finish before sending IO
46102024-02-06T22:17:32.943Zv":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044914157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46112024-02-06T22:17:32.943Z{"msg":"[2]R flush_numbers[0..12]: [81, 81, 81, 79, 82, 77, 76, 83, 83, 83, 82, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044928785Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46122024-02-06T22:17:32.943Z{"msg":"[2]R generation[0..12]: [39, 39, 39, 37, 40, 36, 35, 41, 41, 41, 40, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044949472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46132024-02-06T22:17:32.943Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.04496441Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46142024-02-06T22:17:32.943Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.044985597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46152024-02-06T22:17:32.943Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045000405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46162024-02-06T22:17:32.943Z{"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045017933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46172024-02-06T22:17:32.943Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045044719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46182024-02-06T22:17:32.943Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045069316Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46192024-02-06T22:17:32.943Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045097862Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46202024-02-06T22:17:32.943Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045121649Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46212024-02-06T22:17:32.943Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045148645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46222024-02-06T22:17:32.943Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045180031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46232024-02-06T22:17:32.943Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045197339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46242024-02-06T22:17:32.943Z{"msg":"12de417d-56a8-4c6e-a751-5d3ca9049c61 is now active with session: 5474e5dd-8914-4f9a-b7dc-c79cd79d0863","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045224045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46252024-02-06T22:17:32.943Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.045243352Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303,"session_id":"5474e5dd-8914-4f9a-b7dc-c79cd79d0863"}
46262024-02-06T22:17:32.944Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:32.04526245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303}
46272024-02-06T22:17:32.944ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
46282024-02-06T22:17:32.944ZLoading write count information from file "/tmp/test_repair_verify.data"
46292024-02-06T22:17:32.944ZRun Repair workload
46302024-02-06T22:17:32.944Z01/30 Write block 572 len 28672 data: 7 6 6 6 7 7 6
46312024-02-06T22:17:32.944Z02/30 Write block 343 len 40960 data: 4 4 4 5 5 5 4 6 8 8
46322024-02-06T22:17:32.944Z03/30 Read block 16 len 24576
46332024-02-06T22:17:32.944Z04/30 Write block 132 len 28672 data: 3 3 3 4 4 5 5
46342024-02-06T22:17:32.944Z05/30 Write block 105 len 16384 data: 4 3 5 6
46352024-02-06T22:17:32.944Z06/30 Write block 275 len 12288 data: 3 4 3
46362024-02-06T22:17:32.944Z07/30 Read block 446 len 20480
46372024-02-06T22:17:32.944Z08/30 Write block 196 len 28672 data: 4 5 4 4 4 3 6
46382024-02-06T22:17:32.944Z09/30 Read block 381 len 12288
46392024-02-06T22:17:32.944Z10/30 Read block 162 len 28672
46402024-02-06T22:17:32.944Z11/30 Write block 60 len 12288 data: 6 5 3
46412024-02-06T22:17:32.944Z12/30 Read block 98 len 12288
46422024-02-06T22:17:32.944Z13/30 Read block 310 len 12288
46432024-02-06T22:17:32.944Z14/30 Write block 325 len 4096 data: 3
46442024-02-06T22:17:32.944Z15/30 Write block 347 len 28672 data: 6 6 5 7 9 9 6
46452024-02-06T22:17:32.944Z16/30 Read block 271 len 4096
46462024-02-06T22:17:32.944Z17/30 Read block 76 len 36864
46472024-02-06T22:17:32.944Z18/30 Read block 193 len 16384
46482024-02-06T22:17:32.944Z19/30 Write block 206 len 40960 data: 6 5 6 8 7 5 5 5 3 4
46492024-02-06T22:17:32.944Z20/30 Read block 61 len 28672
46502024-02-06T22:17:32.945Z21/30 Write block 308 len 40960 data: 3 3 4 4 5 6 7 7 5 4
46512024-02-06T22:17:32.945Z22/30 Read block 54 len 24576
46522024-02-06T22:17:32.945Z23/30 Write block 231 len 36864 data: 4 4 2 3 3 3 3 3 3
46532024-02-06T22:17:32.945Z24/30 Write block 186 len 24576 data: 7 7 6 4 3 2
46542024-02-06T22:17:32.945Z25/30 Write block 593 len 28672 data: 7 6 7 6 6 6 2
46552024-02-06T22:17:32.945Z26/30 Write block 118 len 4096 data: 8
46562024-02-06T22:17:32.945Z27/30 Write block 77 len 36864 data: 2 2 2 3 3 4 4 4 3
46572024-02-06T22:17:32.945Z28/30 Write block 275 len 40960 data: 4 5 4 4 5 5 5 5 5 5
46582024-02-06T22:17:32.945Z29/30 Read block 531 len 8192
46592024-02-06T22:17:32.945Z30/30 Write block 376 len 20480 data: 7 9 8 7 7
46602024-02-06T22:17:32.945Z----------------------------------------------------------------
46612024-02-06T22:17:32.945Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46622024-02-06T22:17:32.945ZStates: Active Active Active
46632024-02-06T22:17:32.945ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46642024-02-06T22:17:32.945Z 1 Acked 1000 Write 7 Done Done Sent false
46652024-02-06T22:17:32.945Z 2 Acked 1001 Write 10 Done Done Sent false
46662024-02-06T22:17:32.945Z 3 Acked 1002 Read 6 Done Done Sent false
46672024-02-06T22:17:32.945Z 4 Acked 1003 Write 7 Done Done Sent false
46682024-02-06T22:17:32.951Z 5 Acked 1004 Write 4 Done Done Sent false
46692024-02-06T22:17:32.951Z 6 Acked 1005 Write 3 Done Done Sent false
46702024-02-06T22:17:32.951Z 7 Acked 1006 Read 5 Done Done Sent false
46712024-02-06T22:17:32.951Z 8 Acked 1007 Write 7 Done Done Sent false
46722024-02-06T22:17:32.951Z 9 Acked 1008 Read 3 Done Done Sent false
46732024-02-06T22:17:32.951Z 10 Acked 1009 Read 7 Done Done Sent false
46742024-02-06T22:17:32.951Z 11 Acked 1010 Write 3 Done Done Sent false
46752024-02-06T22:17:32.952Z 12 Acked 1011 Read 3 Done Done Sent false
46762024-02-06T22:17:32.952Z 13 Acked 1012 Read 3 Done Done Sent false
46772024-02-06T22:17:32.952Z 14 Acked 1013 Write 1 Done Done Sent false
46782024-02-06T22:17:32.952Z 15 Acked 1014 Write 7 Done Done Sent false
46792024-02-06T22:17:32.952Z 16 Acked 1015 Read 1 Done Done Sent false
46802024-02-06T22:17:32.952Z 17 Acked 1016 Read 9 Done Done Sent false
46812024-02-06T22:17:32.952Z 18 Acked 1017 Read 4 Done Done Sent false
46822024-02-06T22:17:32.952Z 19 Acked 1018 Write 10 Done Done Sent false
46832024-02-06T22:17:32.952Z 20 Acked 1019 Read 7 Done Done Sent false
46842024-02-06T22:17:32.952Z 21 Acked 1020 Write 10 Done Done Sent false
46852024-02-06T22:17:32.952Z 22 Acked 1021 Read 6 Done Done Sent false
46862024-02-06T22:17:32.952Z 23 Acked 1022 Write 9 Done Done Sent false
46872024-02-06T22:17:32.952Z 24 Acked 1023 Write 6 Done Done Sent false
46882024-02-06T22:17:32.952Z 25 Acked 1024 Write 7 Done Done Sent false
46892024-02-06T22:17:32.952Z 26 Acked 1025 Write 1 Done Done Sent false
46902024-02-06T22:17:32.952Z 27 Acked 1026 Write 9 Done Done Sent false
46912024-02-06T22:17:32.952Z 28 Acked 1027 Write 10 Done Done Sent false
46922024-02-06T22:17:32.952Z 29 Acked 1028 Read 2 Done Done Sent false
46932024-02-06T22:17:32.952Z 30 Acked 1029 Write 5 Sent Sent Sent false
46942024-02-06T22:17:32.952Z STATES DS:0 DS:1 DS:2 TOTAL
46952024-02-06T22:17:32.952Z New 0 0 0 0
46962024-02-06T22:17:32.952Z Sent 1 1 30 32
46972024-02-06T22:17:32.952Z Done 29 29 0 58
46982024-02-06T22:17:32.952Z Skipped 0 0 0 0
46992024-02-06T22:17:32.952Z Error 0 0 0 0
47002024-02-06T22:17:32.953ZLast Flush: 0 0 0
47012024-02-06T22:17:32.953ZDownstairs last five completed:
47022024-02-06T22:17:32.953ZUpstairs last five completed: 30 29 28 27 26
47032024-02-06T22:17:32.953Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:32.074845663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1303}
47042024-02-06T22:17:32.953ZWrote out file "/tmp/test_repair_verify.data"
47052024-02-06T22:17:32.953Z
47062024-02-06T22:17:32.953ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
47072024-02-06T22:17:32.953ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47082024-02-06T22:17:32.953Z 0 000-019 39 39 39 81 81 81 F F F
47092024-02-06T22:17:32.953Z 1 020-039 39 39 39 81 81 81 F F F
47102024-02-06T22:17:32.953Z 2 040-059 39 39 39 81 81 81 F F F
47112024-02-06T22:17:32.953Z 3 060-079 37 37 37 79 79 79 T T F <---
47122024-02-06T22:17:32.953Z 4 080-099 40 40 40 82 82 82 T T F <---
47132024-02-06T22:17:32.953Z 5 100-119 36 36 36 77 77 77 T T F <---
47142024-02-06T22:17:32.953Z 6 120-139 35 35 35 76 76 76 T T F <---
47152024-02-06T22:17:32.953Z 7 140-159 41 41 41 83 83 83 F F F
47162024-02-06T22:17:32.953Z 8 160-179 41 41 41 83 83 83 F F F
47172024-02-06T22:17:32.953Z 9 180-199 41 41 41 83 83 83 T T F <---
47182024-02-06T22:17:32.953Z 10 200-219 40 40 40 82 82 82 T T F <---
47192024-02-06T22:17:32.953Z 11 220-239 35 35 35 76 76 76 T T F <---
47202024-02-06T22:17:32.953Z 12 240-259 41 41 41 83 83 83 F F F
47212024-02-06T22:17:32.953Z 13 260-279 41 41 41 83 83 83 T T F <---
47222024-02-06T22:17:32.953Z 14 280-299 37 37 37 79 79 79 T T F <---
47232024-02-06T22:17:32.953Z 15 300-319 29 29 29 60 60 60 T T F <---
47242024-02-06T22:17:32.954Z 16 320-339 36 36 36 77 77 77 T T F <---
47252024-02-06T22:17:32.954Z 17 340-359 36 36 36 77 77 77 T T F <---
47262024-02-06T22:17:32.954Z 18 360-379 40 40 40 82 82 82 T T F <---
47272024-02-06T22:17:32.954Z 19 380-399 40 40 40 82 82 82 T T F <---
47282024-02-06T22:17:32.954Z 20 400-419 39 39 39 81 81 81 F F F
47292024-02-06T22:17:32.954Z 21 420-439 40 40 40 82 82 82 F F F
47302024-02-06T22:17:32.954Z 22 440-459 34 34 34 73 73 73 F F F
47312024-02-06T22:17:32.954Z 23 460-479 41 41 41 83 83 83 F F F
47322024-02-06T22:17:32.954Z 24 480-499 32 32 32 65 65 65 F F F
47332024-02-06T22:17:32.954Z 25 500-519 41 41 41 83 83 83 F F F
47342024-02-06T22:17:32.954Z 26 520-539 34 34 34 71 71 71 F F F
47352024-02-06T22:17:32.954Z 27 540-559 40 40 40 82 82 82 F F F
47362024-02-06T22:17:32.954Z 28 560-579 40 40 40 82 82 82 T T F <---
47372024-02-06T22:17:32.954Z 29 580-599 41 41 41 83 83 83 T T F <---
47382024-02-06T22:17:32.954ZMax gen: 41, Max flush: 83
47392024-02-06T22:17:32.954ZError: Difference in extent metadata found!
47402024-02-06T22:17:32.954ZA Difference in extent metadata is expected here
47412024-02-06T22:17:32.954ZOn loop 21
47422024-02-06T22:17:32.954Z
47432024-02-06T22:17:32.954ZVerifying data now
47442024-02-06T22:17:37.672ZNo change after verify
47452024-02-06T22:17:37.672ZLoop: 21 Downstairs dump after verify (and repair):
47462024-02-06T22:17:37.687ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47472024-02-06T22:17:37.687Z 0 000-019 39 39 39 81 81 81 F F F
47482024-02-06T22:17:37.687Z 1 020-039 39 39 39 81 81 81 F F F
47492024-02-06T22:17:37.687Z 2 040-059 39 39 39 81 81 81 F F F
47502024-02-06T22:17:37.687Z 3 060-079 42 42 42 84 84 84 F F F
47512024-02-06T22:17:37.688Z 4 080-099 42 42 42 84 84 84 F F F
47522024-02-06T22:17:37.688Z 5 100-119 42 42 42 84 84 84 F F F
47532024-02-06T22:17:37.688Z 6 120-139 42 42 42 84 84 84 F F F
47542024-02-06T22:17:37.688Z 7 140-159 41 41 41 83 83 83 F F F
47552024-02-06T22:17:37.688Z 8 160-179 41 41 41 83 83 83 F F F
47562024-02-06T22:17:37.688Z 9 180-199 42 42 42 84 84 84 F F F
47572024-02-06T22:17:37.688Z 10 200-219 42 42 42 84 84 84 F F F
47582024-02-06T22:17:37.688Z 11 220-239 42 42 42 84 84 84 F F F
47592024-02-06T22:17:37.688Z 12 240-259 41 41 41 83 83 83 F F F
47602024-02-06T22:17:37.688Z 13 260-279 42 42 42 84 84 84 F F F
47612024-02-06T22:17:37.688Z 14 280-299 42 42 42 84 84 84 F F F
47622024-02-06T22:17:37.688Z 15 300-319 42 42 42 84 84 84 F F F
47632024-02-06T22:17:37.688Z 16 320-339 42 42 42 84 84 84 F F F
47642024-02-06T22:17:37.691Z 17 340-359 42 42 42 84 84 84 F F F
47652024-02-06T22:17:37.691Z 18 360-379 42 42 42 84 84 84 F F F
47662024-02-06T22:17:37.691Z 19 380-399 42 42 42 84 84 84 F F F
47672024-02-06T22:17:37.692Z 20 400-419 39 39 39 81 81 81 F F F
47682024-02-06T22:17:37.692Z 21 420-439 40 40 40 82 82 82 F F F
47692024-02-06T22:17:37.692Z 22 440-459 34 34 34 73 73 73 F F F
47702024-02-06T22:17:37.692Z 23 460-479 41 41 41 83 83 83 F F F
47712024-02-06T22:17:37.692Z 24 480-499 32 32 32 65 65 65 F F F
47722024-02-06T22:17:37.692Z 25 500-519 41 41 41 83 83 83 F F F
47732024-02-06T22:17:37.692Z 26 520-539 34 34 34 71 71 71 F F F
47742024-02-06T22:17:37.692Z 27 540-559 40 40 40 82 82 82 F F F
47752024-02-06T22:17:37.692Z 28 560-579 42 42 42 84 84 84 F F F
47762024-02-06T22:17:37.692Z 29 580-599 42 42 42 84 84 84 F F F
47772024-02-06T22:17:37.692ZMax gen: 42, Max flush: 84
47782024-02-06T22:17:37.692Z
47792024-02-06T22:17:37.692ZBegin loop 22
47802024-02-06T22:17:37.692ZDownstairs to restart: 0
47812024-02-06T22:17:37.709Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.93808267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311}
47822024-02-06T22:17:37.709Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.93830624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311}
47832024-02-06T22:17:37.713Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938356653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311}
47842024-02-06T22:17:37.713Z{"msg":"Crucible c3ed98cf-3978-41bf-ba53-a34238802ee8 has session id: 9e3f2f3f-5d77-4b3a-8728-0baa347a5708","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938389089Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47852024-02-06T22:17:37.714Z{"msg":"Upstairs opts: Upstairs UUID: c3ed98cf-3978-41bf-ba53-a34238802ee8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938441082Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47862024-02-06T22:17:37.714Z{"msg":"Crucible stats registered with UUID: c3ed98cf-3978-41bf-ba53-a34238802ee8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938467878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":Crucible runtime is spawned
47872024-02-06T22:17:37.714Z1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47882024-02-06T22:17:37.714Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938570254Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311}
47892024-02-06T22:17:37.714Z{"msg":"c3ed98cf-3978-41bf-ba53-a34238802ee8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938688568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47902024-02-06T22:17:37.714Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938711085Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47912024-02-06T22:17:37.714Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938726993Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47922024-02-06T22:17:37.714Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938757369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47932024-02-06T22:17:37.714Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938772187Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47942024-02-06T22:17:37.714Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938788635Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47952024-02-06T22:17:37.714Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.93882341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47962024-02-06T22:17:37.714Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938863255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47972024-02-06T22:17:37.714Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.9389015Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47982024-02-06T22:17:37.714Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.938941904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
47992024-02-06T22:17:37.714Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.93897043Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48002024-02-06T22:17:37.714Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.939030612Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48012024-02-06T22:17:37.714Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.939073036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48022024-02-06T22:17:37.714Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.939716349Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48032024-02-06T22:17:37.714Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.939763843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48042024-02-06T22:17:37.714Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.939799478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48052024-02-06T22:17:37.714Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.939824055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48062024-02-06T22:17:37.714Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.940332026Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48072024-02-06T22:17:37.714Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.940438332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48082024-02-06T22:17:37.715Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.940793784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48092024-02-06T22:17:37.715Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.940872173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48102024-02-06T22:17:37.715Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.940940584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48112024-02-06T22:17:37.715Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.94097505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48122024-02-06T22:17:37.715Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941010655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48132024-02-06T22:17:37.715Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941028302Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48142024-02-06T22:17:37.715Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941059288Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48152024-02-06T22:17:37.715Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941086684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48162024-02-06T22:17:37.720Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941315284Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48172024-02-06T22:17:37.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941747345Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48182024-02-06T22:17:37.720Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.94178884Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48192024-02-06T22:17:37.720Z{"msg":"[0]R flush_numbers[0..12]: [81, 81, 81, 84, 84, 84, 84, 83, 83, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941805977Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48202024-02-06T22:17:37.720Z{"msg":"[0]R generation[0..12]: [39, 39, 39, 42, 42, 42, 42, 41, 41, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941828084Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48212024-02-06T22:17:37.720Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941843222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48222024-02-06T22:17:37.720Z{"msg":"[1]R flush_numbers[0..12]: [81, 81, 81, 84, 84, 84, 84, 83, 83, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941864039Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48232024-02-06T22:17:37.720Z{"msg":"[1]R generation[0..12]: [39, 39, 39, 42, 42, 42, 42, 41, 41, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941878468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48242024-02-06T22:17:37.720Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941899554Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48252024-02-06T22:17:37.720Z{"msg":"[2]R flush_numbers[0..12]: [81, 81, 81, 84, 84, 84, 84, 83, 83, 84, 84, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941914312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48262024-02-06T22:17:37.720Z{"msg":"[2]R generation[0..12]: [39, 39, 39, 42, 42, 42, 42, 41, 41, 42, 42, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.94193473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48272024-02-06T22:17:37.720Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
48282024-02-06T22:17:37.720Z,"time":"2024-02-06T22:17:36.941949118Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48292024-02-06T22:17:37.720Z{"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941965036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48302024-02-06T22:17:37.720Z{"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.941983973Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48312024-02-06T22:17:37.720Z{"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942018438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48322024-02-06T22:17:37.720Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942036316Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48332024-02-06T22:17:37.721Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942058683Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48342024-02-06T22:17:37.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942071341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48352024-02-06T22:17:37.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942086119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48362024-02-06T22:17:37.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942106237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48372024-02-06T22:17:37.721Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942119925Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48382024-02-06T22:17:37.721Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942132913Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48392024-02-06T22:17:37.721Z{"msg":"c3ed98cf-3978-41bf-ba53-a34238802ee8 is now active with session: 9e3f2f3f-5d77-4b3a-8728-0baa347a5708","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942146041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48402024-02-06T22:17:37.721Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.94215808Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311,"session_id":"9e3f2f3f-5d77-4b3a-8728-0baa347a5708"}
48412024-02-06T22:17:37.721Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:36.942181287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311}
48422024-02-06T22:17:37.721ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48432024-02-06T22:17:37.721ZLoading write count information from file "/tmp/test_repair_verify.data"
48442024-02-06T22:17:37.721ZRun Repair workload
48452024-02-06T22:17:37.721Z01/30 Write block 521 len 40960 data: 3 4 3 3 3 4 4 4 3 4
48462024-02-06T22:17:37.721Z02/30 Read block 263 len 36864
48472024-02-06T22:17:37.721Z03/30 Flush
48482024-02-06T22:17:37.724Z04/30 Write block 128 len 12288 data: 7 5 5
48492024-02-06T22:17:37.724Z05/30 Read block 196 len 24576
48502024-02-06T22:17:37.724Z06/30 Write block 378 len 16384 data: 9 8 8 5
48512024-02-06T22:17:37.724Z07/30 Read block 337 len 24576
48522024-02-06T22:17:37.724Z08/30 Write block 305 len 12288 data: 2 2 2
48532024-02-06T22:17:37.725Z09/30 Write block 367 len 28672 data: 7 6 6 5 5 4 4
48542024-02-06T22:17:37.725Z10/30 Read block 344 len 20480
48552024-02-06T22:17:37.728Z11/30 Read block 246 len 36864
48562024-02-06T22:17:37.728Z12/30 Write block 56 len 20480 data: 8 7 6 6 7
48572024-02-06T22:17:37.728Z13/30 Flush
48582024-02-06T22:17:37.731Z14/30 Flush
48592024-02-06T22:17:37.731Z15/30 Flush
48602024-02-06T22:17:37.731Z16/30 Write block 162 len 8192 data: 4 5
48612024-02-06T22:17:37.731Z17/30 Write block 177 len 4096 data: 5
48622024-02-06T22:17:37.731Z18/30 Read block 57 len 16384
48632024-02-06T22:17:37.734Z19/30 Write block 200 len 32768 data: 5 4 7 7 7 6 7 6
48642024-02-06T22:17:37.734Z20/30 Write block 22 len 8192 data: 6 6
48652024-02-06T22:17:37.734Z21/30 Read block 31 len 8192
48662024-02-06T22:17:37.734Z22/30 Read block 401 len 16384
48672024-02-06T22:17:37.734Z23/30 Read block 274 len 32768
48682024-02-06T22:17:37.737Z24/30 Write block 555 len 16384 data: 3 4 4 5
48692024-02-06T22:17:37.737Z25/30 Flush
48702024-02-06T22:17:37.741Z26/30 Write block 411 len 4096 data: 4
48712024-02-06T22:17:37.741Z27/30 Read block 323 len 8192
48722024-02-06T22:17:37.741Z28/30 Write block 355 len 36864 data: 8 7 9 7 7 7 7 9 8
48732024-02-06T22:17:37.741Z29/30 Read block 233 len 8192
48742024-02-06T22:17:37.744Z30/30 Write block 300 len 28672 data: 5 4 4 2 2 3 3
48752024-02-06T22:17:37.744Z----------------------------------------------------------------
48762024-02-06T22:17:37.744Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48772024-02-06T22:17:37.744ZStates: Active Active Active
48782024-02-06T22:17:37.744ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48792024-02-06T22:17:37.744Z 1 Acked 1000 Write 10 Sent Done Done false
48802024-02-06T22:17:37.745Z 2 Acked 1001 Read 9 Sent Done Done false
48812024-02-06T22:17:37.745Z 3 Acked 1002 Flush 0 Sent Done Done false
48822024-02-06T22:17:37.745Z 4 Acked 1003 Write 3 Sent Done Done false
48832024-02-06T22:17:37.745Z 5 Acked 1004 Read 6 Sent Done Done false
48842024-02-06T22:17:37.745Z 6 Acked 1005 Write 4 Sent Done Done false
48852024-02-06T22:17:37.745Z 7 Acked 1006 Read 6 Sent Done Done false
48862024-02-06T22:17:37.745Z 8 Acked 1007 Write 3 Sent Done Done false
48872024-02-06T22:17:37.745Z 9 Acked 1008 Write 7 Sent Done Done false
48882024-02-06T22:17:37.745Z 10 Acked 1009 Read 5 Sent Done Done false
48892024-02-06T22:17:37.745Z 11 Acked 1010 Read 9 Sent Done Done false
48902024-02-06T22:17:37.745Z 12 Acked 1011 Write 5 Sent Done Done false
48912024-02-06T22:17:37.745Z 13 Acked 1012 Flush 0 Sent Done Done false
48922024-02-06T22:17:37.745Z 14 Acked 1013 Flush 0 Sent Done Done false
48932024-02-06T22:17:37.745Z 15 Acked 1014 Flush 0 Sent Done Done false
48942024-02-06T22:17:37.745Z 16 Acked 1015 Write 2 Sent Done Done false
48952024-02-06T22:17:37.745Z 17 Acked 1016 Write 1 Sent Done Done false
48962024-02-06T22:17:37.745Z 18 Acked 1017 Read 4 Sent Done Done false
48972024-02-06T22:17:37.745Z 19 Acked 1018 Write 8 Sent Done Done false
48982024-02-06T22:17:37.745Z 20 Acked 1019 Write 2 Sent Done Done false
48992024-02-06T22:17:37.745Z 21 Acked 1020 Read 2 Sent Done Done false
49002024-02-06T22:17:37.745Z 22 Acked 1021 Read 4 Sent Done Done false
49012024-02-06T22:17:37.745Z 23 Acked 1022 Read 8 Sent Done Done false
49022024-02-06T22:17:37.745Z 24 Acked 1023 Write 4 Sent Done Done false
49032024-02-06T22:17:37.745Z 25 Acked 1024 Flush 0 Sent Done Done false
49042024-02-06T22:17:37.745Z 26 Acked 1025 Write 1 Sent Done Done false
49052024-02-06T22:17:37.746Z 27 Acked 1026 Read 2 Sent Done Done false
49062024-02-06T22:17:37.746Z 28 Acked 1027 Write 9 Sent Done Done false
49072024-02-06T22:17:37.746Z 29 Acked 1028 Read 2 Sent Done Done false
49082024-02-06T22:17:37.746Z 30 Acked 1029 Write 7 Sent Sent Sent false
49092024-02-06T22:17:37.746Z STATES DS:0 DS:1 DS:2 TOTAL
49102024-02-06T22:17:37.746Z New 0 0 0 0
49112024-02-06T22:17:37.746Z Sent 30 1 1 32
49122024-02-06T22:17:37.746Z Done 0 29 29 58
49132024-02-06T22:17:37.746Z Skipped 0 0 0 0
49142024-02-06T22:17:37.746Z Error 0 0 0 0
49152024-02-06T22:17:37.746ZLast Flush: 0 1024 1024
49162024-02-06T22:17:37.746ZDownstairs last five completed:
49172024-02-06T22:17:37.746ZUpstairs last five completed: 30 29 28 27 26
49182024-02-06T22:17:37.746Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:36.970807129Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1311}
49192024-02-06T22:17:37.746ZWrote out file "/tmp/test_repair_verify.data"
49202024-02-06T22:17:37.749Z
49212024-02-06T22:17:37.749ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
49222024-02-06T22:17:37.770ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49232024-02-06T22:17:37.770Z 0 000-019 39 39 39 81 81 81 F F F
49242024-02-06T22:17:37.771Z 1 020-039 39 44 44 81 89 89 F F F <---
49252024-02-06T22:17:37.771Z 2 040-059 39 44 44 81 86 86 F F F <---
49262024-02-06T22:17:37.771Z 3 060-079 42 44 44 84 86 86 F F F <---
49272024-02-06T22:17:37.771Z 4 080-099 42 42 42 84 84 84 F F F
49282024-02-06T22:17:37.771Z 5 100-119 42 42 42 84 84 84 F F F
49292024-02-06T22:17:37.771Z 6 120-139 42 44 44 84 86 86 F F F <---
49302024-02-06T22:17:37.771Z 7 140-159 41 41 41 83 83 83 F F F
49312024-02-06T22:17:38.184Z 8 160-179 41 44 44 83 89 89 F F F <---
49322024-02-06T22:17:38.184Z 9 180-199 42 42 42 84 84 84 F F F
49332024-02-06T22:17:38.184Z 10 200-219 42 44 44 84 89 89 F F F <---
49342024-02-06T22:17:38.184Z 11 220-239 42 42 42 84 84 84 F F F
49352024-02-06T22:17:38.184Z 12 240-259 41 41 41 83 83 83 F F F
49362024-02-06T22:17:38.184Z 13 260-279 42 42 42 84 84 84 F F F
49372024-02-06T22:17:38.184Z 14 280-299 42 42 42 84 84 84 F F F
49382024-02-06T22:17:38.185Z 15 300-319 42 44 44 84 86 86 F T T <---
49392024-02-06T22:17:38.185Z 16 320-339 42 42 42 84 84 84 F F F
49402024-02-06T22:17:38.185Z 17 340-359 42 42 42 84 84 84 F T T <---
49412024-02-06T22:17:38.185Z 18 360-379 42 44 44 84 86 86 F T T <---
49422024-02-06T22:17:38.185Z 19 380-399 42 44 44 84 86 86 F F F <---
49432024-02-06T22:17:38.185Z 20 400-419 39 39 39 81 81 81 F T T <---
49442024-02-06T22:17:38.185Z 21 420-439 40 40 40 82 82 82 F F F
49452024-02-06T22:17:38.185Z 22 440-459 34 34 34 73 73 73 F F F
49462024-02-06T22:17:38.185Z 23 460-479 41 41 41 83 83 83 F F F
49472024-02-06T22:17:38.185Z 24 480-499 32 32 32 65 65 65 F F F
49482024-02-06T22:17:38.185Z 25 500-519 41 41 41 83 83 83 F F F
49492024-02-06T22:17:38.185Z 26 520-539 34 44 44 71 85 85 F F F <---
49502024-02-06T22:17:38.185Z 27 540-559 40 44 44 82 89 89 F F F <---
49512024-02-06T22:17:38.185Z 28 560-579 42 42 42 84 84 84 F F F
49522024-02-06T22:17:38.185Z 29 580-599 42 42 42 84 84 84 F F F
49532024-02-06T22:17:38.185ZMax gen: 44, Max flush: 89
49542024-02-06T22:17:38.185ZError: Difference in extent metadata found!
49552024-02-06T22:17:38.185ZA Difference in extent metadata is expected here
49562024-02-06T22:17:38.185ZOn loop 22
49572024-02-06T22:17:38.185Z
49582024-02-06T22:17:38.185ZVerifying data now
49592024-02-06T22:17:42.418ZNo change after verify
49602024-02-06T22:17:42.418ZLoop: 22 Downstairs dump after verify (and repair):
49612024-02-06T22:17:42.431ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49622024-02-06T22:17:42.431Z 0 000-019 39 39 39 81 81 81 F F F
49632024-02-06T22:17:42.431Z 1 020-039 44 44 44 89 89 89 F F F
49642024-02-06T22:17:42.432Z 2 040-059 44 44 44 86 86 86 F F F
49652024-02-06T22:17:42.432Z 3 060-079 44 44 44 86 86 86 F F F
49662024-02-06T22:17:42.432Z 4 080-099 42 42 42 84 84 84 F F F
49672024-02-06T22:17:42.432Z 5 100-119 42 42 42 84 84 84 F F F
49682024-02-06T22:17:42.432Z 6 120-139 44 44 44 86 86 86 F F F
49692024-02-06T22:17:42.435Z 7 140-159 41 41 41 83 83 83 F F F
49702024-02-06T22:17:42.435Z 8 160-179 44 44 44 89 89 89 F F F
49712024-02-06T22:17:42.436Z 9 180-199 42 42 42 84 84 84 F F F
49722024-02-06T22:17:42.436Z 10 200-219 44 44 44 89 89 89 F F F
49732024-02-06T22:17:42.436Z 11 220-239 42 42 42 84 84 84 F F F
49742024-02-06T22:17:42.436Z 12 240-259 41 41 41 83 83 83 F F F
49752024-02-06T22:17:42.436Z 13 260-279 42 42 42 84 84 84 F F F
49762024-02-06T22:17:42.436Z 14 280-299 42 42 42 84 84 84 F F F
49772024-02-06T22:17:42.436Z 15 300-319 45 45 45 90 90 90 F F F
49782024-02-06T22:17:42.436Z 16 320-339 42 42 42 84 84 84 F F F
49792024-02-06T22:17:42.436Z 17 340-359 45 45 45 90 90 90 F F F
49802024-02-06T22:17:42.436Z 18 360-379 45 45 45 90 90 90 F F F
49812024-02-06T22:17:42.436Z 19 380-399 44 44 44 86 86 86 F F F
49822024-02-06T22:17:42.436Z 20 400-419 45 45 45 90 90 90 F F F
49832024-02-06T22:17:42.437Z 21 420-439 40 40 40 82 82 82 F F F
49842024-02-06T22:17:42.437Z 22 440-459 34 34 34 73 73 73 F F F
49852024-02-06T22:17:42.437Z 23 460-479 41 41 41 83 83 83 F F F
49862024-02-06T22:17:42.437Z 24 480-499 32 32 32 65 65 65 F F F
49872024-02-06T22:17:42.437Z 25 500-519 41 41 41 83 83 83 F F F
49882024-02-06T22:17:42.437Z 26 520-539 44 44 44 85 85 85 F F F
49892024-02-06T22:17:42.437Z 27 540-559 44 44 44 89 89 89 F F F
49902024-02-06T22:17:42.437Z 28 560-579 42 42 42 84 84 84 F F F
49912024-02-06T22:17:42.437Z 29 580-599 42 42 42 84 84 84 F F F
49922024-02-06T22:17:42.437ZMax gen: 45, Max flush: 90
49932024-02-06T22:17:42.437Z
49942024-02-06T22:17:42.437ZBegin loop 23
49952024-02-06T22:17:42.437ZDownstairs to restart: 1
49962024-02-06T22:17:42.448Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.67591291Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319}
49972024-02-06T22:17:42.456Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible"Crucible runtime is spawned
49982024-02-06T22:17:42.457Z,"level":30,"time":"2024-02-06T22:17:41.676309477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319}
49992024-02-06T22:17:42.457Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676413573Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319}
50002024-02-06T22:17:42.457Z{"msg":"Crucible 96c8d20a-86c3-4232-a42c-87481b1c25bd has session id: 1757d4ad-a453-404f-bd61-356528e1b1d4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676481654Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50012024-02-06T22:17:42.457Z{"msg":"Upstairs opts: Upstairs UUID: 96c8d20a-86c3-4232-a42c-87481b1c25bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676534917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50022024-02-06T22:17:42.457Z{"msg":"Crucible stats registered with UUID: 96c8d20a-86c3-4232-a42c-87481b1c25bd","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676564493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50032024-02-06T22:17:42.457Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676587539Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319}
50042024-02-06T22:17:42.457Z{"msg":"96c8d20a-86c3-4232-a42c-87481b1c25bd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676614936Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50052024-02-06T22:17:42.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676708053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50062024-02-06T22:17:42.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676755537Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50072024-02-06T22:17:42.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676782763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50082024-02-06T22:17:42.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676814729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50092024-02-06T22:17:42.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676837826Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50102024-02-06T22:17:42.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.67688105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50112024-02-06T22:17:42.457Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676905887Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50122024-02-06T22:17:42.457Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676944291Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50132024-02-06T22:17:42.457Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.676968208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50142024-02-06T22:17:42.457Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.677001254Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50152024-02-06T22:17:42.457Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.67702559Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50162024-02-06T22:17:42.458Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.677048027Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50172024-02-06T22:17:42.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.677527072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50182024-02-06T22:17:42.458Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.677641717Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50192024-02-06T22:17:42.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.677685351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50202024-02-06T22:17:42.458Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.677702629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50212024-02-06T22:17:42.458Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678128471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50222024-02-06T22:17:42.458Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.67828141Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50232024-02-06T22:17:42.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678613136Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50242024-02-06T22:17:42.458Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678660969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50252024-02-06T22:17:42.458Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678796611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50262024-02-06T22:17:42.458Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678840665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50272024-02-06T22:17:42.458Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678866551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50282024-02-06T22:17:42.458Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678882559Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50292024-02-06T22:17:42.458Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678896807Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50302024-02-06T22:17:42.458Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.678919874Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50312024-02-06T22:17:42.458Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679178169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50322024-02-06T22:17:42.459Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.67961577Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50332024-02-06T22:17:42.459Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679662334Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50342024-02-06T22:17:42.459Z{"msg":"[0]R flush_numbers[0..12]: [81, 89, 86, 86, 84, 84, 86, 83, 89, 84, 89, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679699169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50352024-02-06T22:17:42.459Z{"msg":"[0]R generation[0..12]: [39, 44, 44, 44, 42, 42, 44, 41, 44, 42, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679734664Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50362024-02-06T22:17:42.459Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679770369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50372024-02-06T22:17:42.459Z{"msg":"[1]R flush_numbers[0..12]: [81, 89, 86, 86, 84, 84, 86, 83, 89, 84, 89, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679807384Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50382024-02-06T22:17:42.459ZWait for a query_work_queue command to finish before sending IO
50392024-02-06T22:17:42.459Z{"msg":"[1]R generation[0..12]: [39, 44, 44, 44, 42, 42, 44, 41, 44, 42, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679849959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50402024-02-06T22:17:42.459Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679890773Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50412024-02-06T22:17:42.459Z{"msg":"[2]R flush_numbers[0..12]: [81, 89, 86, 86, 84, 84, 86, 83, 89, 84, 89, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679930538Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50422024-02-06T22:17:42.459Z{"msg":"[2]R generation[0..12]: [39, 44, 44, 44, 42, 42, 44, 41, 44, 42, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.679968023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50432024-02-06T22:17:42.459Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680023125Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50442024-02-06T22:17:42.459Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.68006134Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50452024-02-06T22:17:42.459Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680100075Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50462024-02-06T22:17:42.459Z{"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.68013753Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50472024-02-06T22:17:42.459Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680175215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50482024-02-06T22:17:42.459Z,"":"downstairs","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50492024-02-06T22:17:42.460Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680217719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50502024-02-06T22:17:42.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680246155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50512024-02-06T22:17:42.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680262363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50522024-02-06T22:17:42.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.68028298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50532024-02-06T22:17:42.460Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680297388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50542024-02-06T22:17:42.460Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680317926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50552024-02-06T22:17:42.460Z{"msg":"96c8d20a-86c3-4232-a42c-87481b1c25bd is now active with session: 1757d4ad-a453-404f-bd61-356528e1b1d4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680331604Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50562024-02-06T22:17:42.460Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680351031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319,"session_id":"1757d4ad-a453-404f-bd61-356528e1b1d4"}
50572024-02-06T22:17:42.460Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:41.680363979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319}
50582024-02-06T22:17:42.460ZLoading write count information from file "/tmp/test_repair_verify.data"
50592024-02-06T22:17:42.460ZRun Repair workload
50602024-02-06T22:17:42.464Z01/30 Write block 209 len 12288 data: 9 8 6
50612024-02-06T22:17:42.464Z02/30 Read block 213 len 24576
50622024-02-06T22:17:42.464Z03/30 Write block 33 len 8192 data: 2 2
50632024-02-06T22:17:42.464Z04/30 Write block 72 len 4096 data: 4
50642024-02-06T22:17:42.464Z05/30 Write block 529 len 12288 data: 4 5 5
50652024-02-06T22:17:42.464Z06/30 Write block 416 len 16384 data: 5 7 7 7
50662024-02-06T22:17:42.464Z07/30 Write block 179 len 8192 data: 7 7
50672024-02-06T22:17:42.464Z08/30 Read block 363 len 36864
50682024-02-06T22:17:42.464Z09/30 Read block 145 len 16384
50692024-02-06T22:17:42.464Z10/30 Write block 305 len 40960 data: 4 4 3 4 4 5 5 6 7 8
50702024-02-06T22:17:42.464Z11/30 Write block 184 len 12288 data: 9 9 8
50712024-02-06T22:17:42.464Z12/30 Read block 221 len 20480
50722024-02-06T22:17:42.464Z13/30 Read block 406 len 8192
50732024-02-06T22:17:42.465Z14/30 Read block 486 len 32768
50742024-02-06T22:17:42.470Z15/30 Write block 118 len 28672 data: 9 8 6 6 6 5 5
50752024-02-06T22:17:42.470Z16/30 Write block 192 len 32768 data: 2 2 4 4 5 6 5 5
50762024-02-06T22:17:42.470Z17/30 Read block 494 len 4096
50772024-02-06T22:17:42.470Z18/30 Write block 593 len 12288 data: 8 7 8
50782024-02-06T22:17:42.471Z19/30 Read block 321 len 16384
50792024-02-06T22:17:42.471Z20/30 Write block 341 len 8192 data: 5 6
50802024-02-06T22:17:42.471Z21/30 Read block 244 len 24576
50812024-02-06T22:17:42.471Z22/30 Read block 181 len 4096
50822024-02-06T22:17:42.471Z23/30 Read block 66 len 32768
50832024-02-06T22:17:42.475Z24/30 Read block 222 len 12288
50842024-02-06T22:17:42.475Z25/30 Write block 564 len 36864 data: 5 5 4 6 6 5 5 7 8
50852024-02-06T22:17:42.475Z26/30 Read block 304 len 40960
50862024-02-06T22:17:42.475Z27/30 Read block 7 len 32768
50872024-02-06T22:17:42.479Z28/30 Write block 155 len 12288 data: 8 7 8
50882024-02-06T22:17:42.479Z29/30 Read block 575 len 24576
50892024-02-06T22:17:42.479Z30/30 Read block 249 len 40960
50902024-02-06T22:17:42.479Z----------------------------------------------------------------
50912024-02-06T22:17:42.479Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50922024-02-06T22:17:42.479ZStates: Active Active Active
50932024-02-06T22:17:42.479ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50942024-02-06T22:17:42.479Z 1 Acked 1000 Write 3 Done Done Done false
50952024-02-06T22:17:42.479Z 2 Acked 1001 Read 6 Done Done Done false
50962024-02-06T22:17:42.480Z 3 Acked 1002 Write 2 Done Done Done false
50972024-02-06T22:17:42.480Z 4 Acked 1003 Write 1 Done Done Done false
50982024-02-06T22:17:42.480Z 5 Acked 1004 Write 3 Done Done Done false
50992024-02-06T22:17:42.480Z 6 Acked 1005 Write 4 Done Done Done false
51002024-02-06T22:17:42.480Z 7 Acked 1006 Write 2 Done Sent Done false
51012024-02-06T22:17:42.480Z 8 Acked 1007 Read 9 Done Sent Done false
51022024-02-06T22:17:42.480Z 9 Acked 1008 Read 4 Done Sent Done false
51032024-02-06T22:17:42.480Z 10 Acked 1009 Write 10 Done Sent Done false
51042024-02-06T22:17:42.480Z 11 Acked 1010 Write 3 Done Sent Done false
51052024-02-06T22:17:42.480Z 12 Acked 1011 Read 5 Done Sent Done false
51062024-02-06T22:17:42.481Z 13 Acked 1012 Read 2 Done Sent Done false
51072024-02-06T22:17:42.481Z 14 Acked 1013 Read 8 Done Sent Done false
51082024-02-06T22:17:42.481Z 15 Acked 1014 Write 7 Done Sent Done false
51092024-02-06T22:17:42.481Z 16 Acked 1015 Write 8 Done Sent Done false
51102024-02-06T22:17:42.481Z 17 Acked 1016 Read 1 Done Sent Done false
51112024-02-06T22:17:42.481Z 18 Acked 1017 Write 3 Done Sent Done false
51122024-02-06T22:17:42.481Z 19 Acked 1018 Read 4 Done Sent Done false
51132024-02-06T22:17:42.481Z 20 Acked 1019 Write 2 Done Sent Done false
51142024-02-06T22:17:42.482Z 21 Acked 1020 Read 6 Done Sent Done false
51152024-02-06T22:17:42.482Z 22 Acked 1021 Read 1 Done Sent Done false
51162024-02-06T22:17:42.482Z 23 Acked 1022 Read 8 Done Sent Done false
51172024-02-06T22:17:42.482Z 24 Acked 1023 Read 3 Done Sent Done false
51182024-02-06T22:17:42.482Z 25 Acked 1024 Write 9 Done Sent Done false
51192024-02-06T22:17:42.482Z 26 Acked 1025 Read 10 Done Sent Done false
51202024-02-06T22:17:42.482Z 27 Acked 1026 Read 8 Done Sent Done false
51212024-02-06T22:17:42.482Z 28 Acked 1027 Write 3 Done Sent Done false
51222024-02-06T22:17:42.482Z 29 Acked 1028 Read 6 Done Sent Done false
51232024-02-06T22:17:42.482Z 30 Acked 1029 Read 10 Done Sent Done false
51242024-02-06T22:17:42.482Z STATES DS:0 DS:1 DS:2 TOTAL
51252024-02-06T22:17:42.483Z New 0 0 0 0
51262024-02-06T22:17:42.483Z Sent 0 24 0 24
51272024-02-06T22:17:42.483Z Done 30 6 30 66
51282024-02-06T22:17:42.483Z Skipped 0 0 0 0
51292024-02-06T22:17:42.483Z Error 0 0 0 0
51302024-02-06T22:17:42.483ZLast Flush: 0 0 0
51312024-02-06T22:17:42.483ZDownstairs last five completed:
51322024-02-06T22:17:42.483ZUpstairs last five completed: 30 29 28 27 26
51332024-02-06T22:17:42.483Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:41.706345929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1319}
51342024-02-06T22:17:42.488ZWrote out file "/tmp/test_repair_verify.data"
51352024-02-06T22:17:42.488Z
51362024-02-06T22:17:42.488ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51372024-02-06T22:17:42.511ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51382024-02-06T22:17:42.511Z 0 000-019 39 39 39 81 81 81 F F F
51392024-02-06T22:17:42.511Z 1 020-039 44 44 44 89 89 89 T T T
51402024-02-06T22:17:42.511Z 2 040-059 44 44 44 86 86 86 F F F
51412024-02-06T22:17:42.512Z 3 060-079 44 44 44 86 86 86 T T T
51422024-02-06T22:17:42.512Z 4 080-099 42 42 42 84 84 84 F F F
51432024-02-06T22:17:42.512Z 5 100-119 42 42 42 84 84 84 T F T <---
51442024-02-06T22:17:42.512Z 6 120-139 44 44 44 86 86 86 T F T <---
51452024-02-06T22:17:42.519Z 7 140-159 41 41 41 83 83 83 T F T <---
51462024-02-06T22:17:42.519Z 8 160-179 44 44 44 89 89 89 T F T <---
51472024-02-06T22:17:42.519Z 9 180-199 42 42 42 84 84 84 T F T <---
51482024-02-06T22:17:42.520Z 10 200-219 44 44 44 89 89 89 T T T
51492024-02-06T22:17:42.520Z 11 220-239 42 42 42 84 84 84 F F F
51502024-02-06T22:17:42.520Z 12 240-259 41 41 41 83 83 83 F F F
51512024-02-06T22:17:42.520Z 13 260-279 42 42 42 84 84 84 F F F
51522024-02-06T22:17:42.520Z 14 280-299 42 42 42 84 84 84 F F F
51532024-02-06T22:17:42.520Z 15 300-319 45 45 45 90 90 90 T F T <---
51542024-02-06T22:17:42.520Z 16 320-339 42 42 42 84 84 84 F F F
51552024-02-06T22:17:42.520Z 17 340-359 45 45 45 90 90 90 T F T <---
51562024-02-06T22:17:42.521Z 18 360-379 45 45 45 90 90 90 F F F
51572024-02-06T22:17:42.521Z 19 380-399 44 44 44 86 86 86 F F F
51582024-02-06T22:17:42.521Z 20 400-419 45 45 45 90 90 90 T T T
51592024-02-06T22:17:42.521Z 21 420-439 40 40 40 82 82 82 F F F
51602024-02-06T22:17:42.521Z 22 440-459 34 34 34 73 73 73 F F F
51612024-02-06T22:17:42.521Z 23 460-479 41 41 41 83 83 83 F F F
51622024-02-06T22:17:42.521Z 24 480-499 32 32 32 65 65 65 F F F
51632024-02-06T22:17:42.521Z 25 500-519 41 41 41 83 83 83 F F F
51642024-02-06T22:17:42.521Z 26 520-539 44 44 44 85 85 85 T T T
51652024-02-06T22:17:42.521Z 27 540-559 44 44 44 89 89 89 F F F
51662024-02-06T22:17:42.521Z 28 560-579 42 42 42 84 84 84 T F T <---
51672024-02-06T22:17:42.521Z 29 580-599 42 42 42 84 84 84 T F T <---
51682024-02-06T22:17:42.521ZMax gen: 45, Max flush: 90
51692024-02-06T22:17:42.522ZError: Difference in extent metadata found!
51702024-02-06T22:17:42.522ZA Difference in extent metadata is expected here
51712024-02-06T22:17:42.522ZOn loop 23
51722024-02-06T22:17:42.522Z
51732024-02-06T22:17:42.522ZVerifying data now
51742024-02-06T22:18:01.967ZNo change after verify
51752024-02-06T22:18:01.967ZLoop: 23 Downstairs dump after verify (and repair):
51762024-02-06T22:18:02.895ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51772024-02-06T22:18:02.895Z 0 000-019 39 39 39 81 81 81 F F F
51782024-02-06T22:18:02.895Z 1 020-039 46 46 46 91 91 91 F F F
51792024-02-06T22:18:02.895Z 2 040-059 44 44 44 86 86 86 F F F
51802024-02-06T22:18:02.896Z 3 060-079 46 46 46 91 91 91 F F F
51812024-02-06T22:18:02.896Z 4 080-099 42 42 42 84 84 84 F F F
51822024-02-06T22:18:02.896Z 5 100-119 46 46 46 91 91 91 F F F
51832024-02-06T22:18:02.896Z 6 120-139 46 46 46 91 91 91 F F F
51842024-02-06T22:18:02.896Z 7 140-159 46 46 46 91 91 91 F F F
51852024-02-06T22:18:02.896Z 8 160-179 46 46 46 91 91 91 F F F
51862024-02-06T22:18:02.896Z 9 180-199 46 46 46 91 91 91 F F F
51872024-02-06T22:18:02.896Z 10 200-219 46 46 46 91 91 91 F F F
51882024-02-06T22:18:02.896Z 11 220-239 42 42 42 84 84 84 F F F
51892024-02-06T22:18:02.896Z 12 240-259 41 41 41 83 83 83 F F F
51902024-02-06T22:18:02.896Z 13 260-279 42 42 42 84 84 84 F F F
51912024-02-06T22:18:02.896Z 14 280-299 42 42 42 84 84 84 F F F
51922024-02-06T22:18:02.896Z 15 300-319 46 46 46 91 91 91 F F F
51932024-02-06T22:18:02.896Z 16 320-339 42 42 42 84 84 84 F F F
51942024-02-06T22:18:02.896Z 17 340-359 46 46 46 91 91 91 F F F
51952024-02-06T22:18:02.896Z 18 360-379 45 45 45 90 90 90 F F F
51962024-02-06T22:18:02.896Z 19 380-399 44 44 44 86 86 86 F F F
51972024-02-06T22:18:02.896Z 20 400-419 46 46 46 91 91 91 F F F
51982024-02-06T22:18:02.896Z 21 420-439 40 40 40 82 82 82 F F F
51992024-02-06T22:18:02.896Z 22 440-459 34 34 34 73 73 73 F F F
52002024-02-06T22:18:02.897Z 23 460-479 41 41 41 83 83 83 F F F
52012024-02-06T22:18:02.897Z 24 480-499 32 32 32 65 65 65 F F F
52022024-02-06T22:18:02.897Z 25 500-519 41 41 41 83 83 83 F F F
52032024-02-06T22:18:02.897Z 26 520-539 46 46 46 91 91 91 F F F
52042024-02-06T22:18:02.897Z 27 540-559 44 44 44 89 89 89 F F F
52052024-02-06T22:18:02.897Z 28 560-579 46 46 46 91 91 91 F F F
52062024-02-06T22:18:02.897Z 29 580-599 46 46 46 91 91 91 F F F
52072024-02-06T22:18:02.897ZMax gen: 46, Max flush: 91
52082024-02-06T22:18:02.897Z
52092024-02-06T22:18:02.897ZBegin loop 24
52102024-02-06T22:18:02.897ZDownstairs to restart: 1
52112024-02-06T22:18:02.897Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.476604557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327}
52122024-02-06T22:18:02.897Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.476872931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327}
52132024-02-06T22:18:02.897Z{"Crucible runtime is spawned
52142024-02-06T22:18:02.897Zmsg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.476926313Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327}
52152024-02-06T22:18:02.897Z{"msg":"Crucible a10fcffe-05c2-432d-8c71-acb9662fc88b has session id: 6c08514a-695b-4dd1-ab57-9d82c97b0bc4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.476960499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52162024-02-06T22:18:02.897Z{"msg":"Upstairs opts: Upstairs UUID: a10fcffe-05c2-432d-8c71-acb9662fc88b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477016061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52172024-02-06T22:18:02.897Z{"msg":"Crucible stats registered with UUID: a10fcffe-05c2-432d-8c71-acb9662fc88b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477049367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52182024-02-06T22:18:02.898Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477076823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327}
52192024-02-06T22:18:02.898Z{"msg":"a10fcffe-05c2-432d-8c71-acb9662fc88b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.4770974Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52202024-02-06T22:18:02.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.47717536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52212024-02-06T22:18:02.898Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477228173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52222024-02-06T22:18:02.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477262188Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52232024-02-06T22:18:02.898Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477305742Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52242024-02-06T22:18:02.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477335678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52252024-02-06T22:18:02.898Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477371333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52262024-02-06T22:18:02.898Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.4773973Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52272024-02-06T22:18:02.898Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477425416Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52282024-02-06T22:18:02.898Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477451183Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52292024-02-06T22:18:02.898Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477478069Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52302024-02-06T22:18:02.898Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.477515434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52312024-02-06T22:18:02.898Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.47754058Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52322024-02-06T22:18:02.898Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.478160367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52332024-02-06T22:18:02.898Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.47820752Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52342024-02-06T22:18:02.898Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.478239596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52352024-02-06T22:18:02.898Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.47828558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52362024-02-06T22:18:02.898Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.47887458Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52372024-02-06T22:18:02.898Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.478916385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52382024-02-06T22:18:02.898Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479325959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52392024-02-06T22:18:02.898Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479377882Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52402024-02-06T22:18:02.898Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479515254Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52412024-02-06T22:18:02.899Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479549219Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52422024-02-06T22:18:02.899Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479565287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52432024-02-06T22:18:02.899Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479578235Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52442024-02-06T22:18:02.899Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479592463Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52452024-02-06T22:18:02.899Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479606591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52462024-02-06T22:18:02.899Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.479875685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52472024-02-06T22:18:02.899Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480318245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52482024-02-06T22:18:02.899Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480350491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52492024-02-06T22:18:02.899Z{"msg":"[0]R flush_numbers[0..12]: [81, 91, 86, 91, 84, 91, 91, 91, 91, 91, 91, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480367318Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52502024-02-06T22:18:02.899Z{"msg":"[0]R generation[0..12]: [39, 46, 44, 46, 42, 46, 46, 46, 46, 46, 46, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480381907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52512024-02-06T22:18:02.899Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480405643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52522024-02-06T22:18:02.899Z{"msg":"[1]R flush_numbers[0..12]: [81, 91, 86, 91, 84, 91, 91, 91, 91, 91, 91, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480420471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52532024-02-06T22:18:02.899Z{"msg":"[1]R generation[0..12]: [39, 46, 44, 46, 42, 46, 46, 46, 46, 46, 46, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480442638Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52542024-02-06T22:18:02.899Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480457406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52552024-02-06T22:18:02.899Z{"msg":"[2]R flush_numbers[0..12]: [81, 91, 86, 91, 84, 91, 91, 91, 91, 91, 91, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480478294Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52562024-02-06T22:18:02.899Z{"msg":"[2]R generation[0..12]: [39, 46, 44, 46, 42, 46, 46, 46, 46, 46, 46, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480493152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52572024-02-06T22:18:02.899Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
52582024-02-06T22:18:02.899Z,"time":"2024-02-06T22:17:47.480514958Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52592024-02-06T22:18:02.899Z{"msg":"Max found gen is 47","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480528937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52602024-02-06T22:18:02.899Z{"msg":"Generation requested: 48 >= found:47","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480543005Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52612024-02-06T22:18:02.899Z{"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480556453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52622024-02-06T22:18:02.899Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480569831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52632024-02-06T22:18:02.899Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480584159Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52642024-02-06T22:18:02.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480603466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52652024-02-06T22:18:02.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480617275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52662024-02-06T22:18:02.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480637152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52672024-02-06T22:18:02.900Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.48065083Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52682024-02-06T22:18:02.900Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480669888Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52692024-02-06T22:18:02.900Z{"msg":"a10fcffe-05c2-432d-8c71-acb9662fc88b is now active with session: 6c08514a-695b-4dd1-ab57-9d82c97b0bc4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480683056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52702024-02-06T22:18:02.900Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480701793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327,"session_id":"6c08514a-695b-4dd1-ab57-9d82c97b0bc4"}
52712024-02-06T22:18:02.900Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:47.480714562Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327}
52722024-02-06T22:18:02.900ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52732024-02-06T22:18:02.900ZLoading write count information from file "/tmp/test_repair_verify.data"
52742024-02-06T22:18:02.900ZRun Repair workload
52752024-02-06T22:18:02.900Z01/30 Flush
52762024-02-06T22:18:02.942Z02/30 Write block 543 len 16384 data: 5 6 5 5
52772024-02-06T22:18:02.942Z03/30 Read block 354 len 20480
52782024-02-06T22:18:02.942Z04/30 Write block 49 len 32768 data: 7 8 10 10 11 11 11 9
52792024-02-06T22:18:02.942Z05/30 Read block 392 len 28672
52802024-02-06T22:18:02.942Z06/30 Write block 377 len 8192 data: 10 10
52812024-02-06T22:18:02.942Z07/30 Read block 385 len 8192
52822024-02-06T22:18:02.942Z08/30 Flush
52832024-02-06T22:18:02.942Z09/30 Write block 66 len 20480 data: 4 5 6 6 5
52842024-02-06T22:18:02.942Z10/30 Read block 29 len 20480
52852024-02-06T22:18:02.943Z11/30 Write block 472 len 28672 data: 6 5 5 5 6 7 7
52862024-02-06T22:18:02.943Z12/30 Read block 58 len 40960
52872024-02-06T22:18:02.943Z13/30 Read block 92 len 20480
52882024-02-06T22:18:02.943Z14/30 Write block 577 len 36864 data: 8 7 5 6 4 3 2 2 3
52892024-02-06T22:18:02.943Z15/30 Read block 420 len 24576
52902024-02-06T22:18:02.943Z16/30 Flush
52912024-02-06T22:18:02.943Z17/30 Write block 408 len 16384 data: 3 4 4 5
52922024-02-06T22:18:02.943Z18/30 Write block 196 len 4096 data: 6
52932024-02-06T22:18:02.943Z19/30 Read block 268 len 36864
52942024-02-06T22:18:02.943Z20/30 Read block 564 len 20480
52952024-02-06T22:18:02.943Z21/30 Flush
52962024-02-06T22:18:02.943Z22/30 Write block 323 len 12288 data: 3 3 4
52972024-02-06T22:18:02.943Z23/30 Write block 363 len 24576 data: 9 7 7 7 8 7
52982024-02-06T22:18:02.943Z24/30 Write block 220 len 20480 data: 5 5 4 4 4
52992024-02-06T22:18:02.943Z25/30 Write block 53 len 24576 data: 12 12 12 10 8 7
53002024-02-06T22:18:02.943Z26/30 Write block 533 len 24576 data: 4 4 2 3 3 3
53012024-02-06T22:18:02.943Z27/30 Write block 400 len 28672 data: 4 5 5 4 4 2 3
53022024-02-06T22:18:02.943Z28/30 Write block 565 len 16384 data: 6 5 7 7
53032024-02-06T22:18:02.943Z29/30 Write block 467 len 20480 data: 4 5 5 5 6
53042024-02-06T22:18:02.943Z30/30 Read block 208 len 28672
53052024-02-06T22:18:02.943Z----------------------------------------------------------------
53062024-02-06T22:18:02.944Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:29
53072024-02-06T22:18:02.944ZStates: Active Active Active
53082024-02-06T22:18:02.944ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
53092024-02-06T22:18:02.944Z 2 Acked 1001 Write 4 Done Done Done false
53102024-02-06T22:18:02.944Z 3 Acked 1002 Read 5 Done Done Done false
53112024-02-06T22:18:02.944Z 4 Acked 1003 Write 8 Done Sent Done false
53122024-02-06T22:18:02.944Z 5 Acked 1004 Read 7 Done Sent Done false
53132024-02-06T22:18:02.944Z 6 Acked 1005 Write 2 Done Sent Done false
53142024-02-06T22:18:02.944Z 7 Acked 1006 Read 2 Done Sent Done false
53152024-02-06T22:18:02.944Z 8 Acked 1007 Flush 0 Done Sent Done false
53162024-02-06T22:18:02.944Z 9 Acked 1008 Write 5 Done Sent Done false
53172024-02-06T22:18:02.944Z 10 Acked 1009 Read 5 Done Sent Done false
53182024-02-06T22:18:02.944Z 11 Acked 1010 Write 7 Done Sent Done false
53192024-02-06T22:18:02.944Z 12 Acked 1011 Read 10 Done Sent Done false
53202024-02-06T22:18:02.944Z 13 Acked 1012 Read 5 Done Sent Done false
53212024-02-06T22:18:02.944Z 14 Acked 1013 Write 9 Done Sent Done false
53222024-02-06T22:18:02.944Z 15 Acked 1014 Read 6 Done Sent Done false
53232024-02-06T22:18:02.944Z 16 Acked 1015 Flush 0 Done Sent Done false
53242024-02-06T22:18:02.944Z 17 Acked 1016 Write 4 Done Sent Done false
53252024-02-06T22:18:02.944Z 18 Acked 1017 Write 1 Done Sent Done false
53262024-02-06T22:18:02.944Z 19 Acked 1018 Read 9 Done Sent Done false
53272024-02-06T22:18:02.944Z 20 Acked 1019 Read 5 Done Sent Done false
53282024-02-06T22:18:02.944Z 21 Acked 1020 Flush 0 Done Sent Done false
53292024-02-06T22:18:02.944Z 22 Acked 1021 Write 3 Done Sent Done false
53302024-02-06T22:18:02.945Z 23 Acked 1022 Write 6 Done Sent Done false
53312024-02-06T22:18:02.945Z 24 Acked 1023 Write 5 Done Sent Done false
53322024-02-06T22:18:02.945Z 25 Acked 1024 Write 6 Done Sent Done false
53332024-02-06T22:18:02.945Z 26 Acked 1025 Write 6 Done Sent Done false
53342024-02-06T22:18:02.945Z 27 Acked 1026 Write 7 Done Sent Done false
53352024-02-06T22:18:02.945Z 28 Acked 1027 Write 4 Done Sent Done false
53362024-02-06T22:18:02.945Z 29 Acked 1028 Write 5 Done Sent Done false
53372024-02-06T22:18:02.945Z 30 Acked 1029 Read 7 Done Sent Sent false
53382024-02-06T22:18:02.945Z STATES DS:0 DS:1 DS:2 TOTAL
53392024-02-06T22:18:02.945Z New 0 0 0 0
53402024-02-06T22:18:02.945Z Sent 0 27 1 28
53412024-02-06T22:18:02.945Z Done 29 2 28 59
53422024-02-06T22:18:02.945Z Skipped 0 0 0 0
53432024-02-06T22:18:02.945Z Error 0 0 0 0
53442024-02-06T22:18:02.945ZLast Flush: 1020 1000 1020
53452024-02-06T22:18:02.945ZDownstairs last five completed: 1000
53462024-02-06T22:18:02.945ZUpstairs last five completed: 30 29 28 27 26
53472024-02-06T22:18:02.945Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:47.510719028Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1327}
53482024-02-06T22:18:02.945ZWrote out file "/tmp/test_repair_verify.data"
53492024-02-06T22:18:02.945Z
53502024-02-06T22:18:02.946ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53512024-02-06T22:18:02.946ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53522024-02-06T22:18:02.946Z 0 000-019 39 39 39 81 81 81 F F F
53532024-02-06T22:18:02.946Z 1 020-039 46 46 46 91 91 91 F F F
53542024-02-06T22:18:02.946Z 2 040-059 48 44 48 93 86 93 T F T <---
53552024-02-06T22:18:02.946Z 3 060-079 48 46 48 94 91 94 F F F <---
53562024-02-06T22:18:02.946Z 4 080-099 42 42 42 84 84 84 F F F
53572024-02-06T22:18:02.946Z 5 100-119 46 46 46 91 91 91 F F F
53582024-02-06T22:18:02.946Z 6 120-139 46 46 46 91 91 91 F F F
53592024-02-06T22:18:02.946Z 7 140-159 46 46 46 91 91 91 F F F
53602024-02-06T22:18:02.946Z 8 160-179 46 46 46 91 91 91 F F F
53612024-02-06T22:18:02.946Z 9 180-199 48 46 48 95 91 95 F F F <---
53622024-02-06T22:18:02.946Z 10 200-219 46 46 46 91 91 91 F F F
53632024-02-06T22:18:02.946Z 11 220-239 42 42 42 84 84 84 T F T <---
53642024-02-06T22:18:02.946Z 12 240-259 41 41 41 83 83 83 F F F
53652024-02-06T22:18:02.946Z 13 260-279 42 42 42 84 84 84 F F F
53662024-02-06T22:18:02.946Z 14 280-299 42 42 42 84 84 84 F F F
53672024-02-06T22:18:02.946Z 15 300-319 46 46 46 91 91 91 F F F
53682024-02-06T22:18:02.946Z 16 320-339 42 42 42 84 84 84 T F T <---
53692024-02-06T22:18:02.946Z 17 340-359 46 46 46 91 91 91 F F F
53702024-02-06T22:18:02.946Z 18 360-379 48 45 48 93 90 93 T F T <---
53712024-02-06T22:18:02.946Z 19 380-399 44 44 44 86 86 86 F F F
53722024-02-06T22:18:02.946Z 20 400-419 48 46 48 95 91 95 T F T <---
53732024-02-06T22:18:02.946Z 21 420-439 40 40 40 82 82 82 F F F
53742024-02-06T22:18:02.946Z 22 440-459 34 34 34 73 73 73 F F F
53752024-02-06T22:18:02.947Z 23 460-479 48 41 48 94 83 94 T F T <---
53762024-02-06T22:18:03.118Z 24 480-499 32 32 32 65 65 65 F F F
53772024-02-06T22:18:03.118Z 25 500-519 41 41 41 83 83 83 F F F
53782024-02-06T22:18:03.118Z 26 520-539 46 46 46 91 91 91 T F T <---
53792024-02-06T22:18:03.118Z 27 540-559 48 44 48 93 89 93 F T F <---
53802024-02-06T22:18:03.119Z 28 560-579 48 46 48 94 91 94 T F T <---
53812024-02-06T22:18:03.119Z 29 580-599 48 46 48 94 91 94 F F F <---
53822024-02-06T22:18:03.119ZMax gen: 48, Max flush: 95
53832024-02-06T22:18:03.119ZError: Difference in extent metadata found!
53842024-02-06T22:18:03.119ZA Difference in extent metadata is expected here
53852024-02-06T22:18:03.119ZOn loop 24
53862024-02-06T22:18:03.119Z
53872024-02-06T22:18:03.119ZVerifying data now
53882024-02-06T22:18:03.120ZNo change after verify
53892024-02-06T22:18:03.120ZLoop: 24 Downstairs dump after verify (and repair):
53902024-02-06T22:18:03.120ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53912024-02-06T22:18:03.120Z 0 000-019 39 39 39 81 81 81 F F F
53922024-02-06T22:18:03.120Z 1 020-039 46 46 46 91 91 91 F F F
53932024-02-06T22:18:03.120Z 2 040-059 49 49 49 96 96 96 F F F
53942024-02-06T22:18:03.121Z 3 060-079 48 48 48 94 94 94 F F F
53952024-02-06T22:18:03.121Z 4 080-099 42 42 42 84 84 84 F F F
53962024-02-06T22:18:03.121Z 5 100-119 46 46 46 91 91 91 F F F
53972024-02-06T22:18:03.121Z 6 120-139 46 46 46 91 91 91 F F F
53982024-02-06T22:18:03.121Z 7 140-159 46 46 46 91 91 91 F F F
53992024-02-06T22:18:03.121Z 8 160-179 46 46 46 91 91 91 F F F
54002024-02-06T22:18:03.121Z 9 180-199 48 48 48 95 95 95 F F F
54012024-02-06T22:18:03.121Z 10 200-219 46 46 46 91 91 91 F F F
54022024-02-06T22:18:03.122Z 11 220-239 49 49 49 96 96 96 F F F
54032024-02-06T22:18:03.122Z 12 240-259 41 41 41 83 83 83 F F F
54042024-02-06T22:18:03.122Z 13 260-279 42 42 42 84 84 84 F F F
54052024-02-06T22:18:03.122Z 14 280-299 42 42 42 84 84 84 F F F
54062024-02-06T22:18:03.122Z 15 300-319 46 46 46 91 91 91 F F F
54072024-02-06T22:18:03.122Z 16 320-339 49 49 49 96 96 96 F F F
54082024-02-06T22:18:03.123Z 17 340-359 46 46 46 91 91 91 F F F
54092024-02-06T22:18:03.123Z 18 360-379 49 49 49 96 96 96 F F F
54102024-02-06T22:18:03.123Z 19 380-399 44 44 44 86 86 86 F F F
54112024-02-06T22:18:03.123Z 20 400-419 49 49 49 96 96 96 F F F
54122024-02-06T22:18:03.123Z 21 420-439 40 40 40 82 82 82 F F F
54132024-02-06T22:18:03.123Z 22 440-459 34 34 34 73 73 73 F F F
54142024-02-06T22:18:03.124Z 23 460-479 49 49 49 96 96 96 F F F
54152024-02-06T22:18:03.124Z 24 480-499 32 32 32 65 65 65 F F F
54162024-02-06T22:18:03.124Z 25 500-519 41 41 41 83 83 83 F F F
54172024-02-06T22:18:03.124Z 26 520-539 49 49 49 96 96 96 F F F
54182024-02-06T22:18:03.124Z 27 540-559 48 48 48 93 93 93 F F F
54192024-02-06T22:18:03.124Z 28 560-579 49 49 49 96 96 96 F F F
54202024-02-06T22:18:03.124Z 29 580-599 48 48 48 94 94 94 F F F
54212024-02-06T22:18:03.124ZMax gen: 49, Max flush: 96
54222024-02-06T22:18:03.124Z
54232024-02-06T22:18:03.124ZBegin loop 25
54242024-02-06T22:18:03.125ZDownstairs to restart: 0
54252024-02-06T22:18:03.125Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.193563207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335}
54262024-02-06T22:18:03.125Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.193821142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335}
54272024-02-06T22:18:03.125Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.193855058Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335}
54282024-02-06T22:18:03.125Z{"msg":"Crucible 7a1ff404-cf30-4348-88d9-5bf91746d6a1 has session id: 62110184-d5a5-4f5a-9e7c-524c3d5b84b6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.193877225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54292024-02-06T22:18:03.125Z{"msg":"Crucible runtime is spawned
54302024-02-06T22:18:03.126ZUpstairs opts: Upstairs UUID: 7a1ff404-cf30-4348-88d9-5bf91746d6a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.193922869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54312024-02-06T22:18:03.126Z{"msg":"Crucible stats registered with UUID: 7a1ff404-cf30-4348-88d9-5bf91746d6a1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.193954614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54322024-02-06T22:18:03.126Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.193978031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335}
54332024-02-06T22:18:03.126Z{"msg":"7a1ff404-cf30-4348-88d9-5bf91746d6a1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194154257Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54342024-02-06T22:18:03.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194181184Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54352024-02-06T22:18:03.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194198511Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54362024-02-06T22:18:03.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194224308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54372024-02-06T22:18:03.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194287549Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54382024-02-06T22:18:03.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194351041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54392024-02-06T22:18:03.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194401814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54402024-02-06T22:18:03.127Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.1944305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54412024-02-06T22:18:03.128Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194468695Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54422024-02-06T22:18:03.128Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194508199Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54432024-02-06T22:18:03.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194532906Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54442024-02-06T22:18:03.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194568062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54452024-02-06T22:18:03.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.194604597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54462024-02-06T22:18:03.128Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.195241531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54472024-02-06T22:18:03.129Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.195289194Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54482024-02-06T22:18:03.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.195315721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54492024-02-06T22:18:03.129Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.195339807Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54502024-02-06T22:18:03.130Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.195962633Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54512024-02-06T22:18:03.130Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196003048Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54522024-02-06T22:18:03.130Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196232877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54532024-02-06T22:18:03.130Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196288649Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54542024-02-06T22:18:03.130Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196515238Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54552024-02-06T22:18:03.131Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196569931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54562024-02-06T22:18:03.131Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196612555Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54572024-02-06T22:18:03.132Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.19665393Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54582024-02-06T22:18:03.132Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196695264Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54592024-02-06T22:18:03.132Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196735878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54602024-02-06T22:18:03.132Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.196894367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54612024-02-06T22:18:03.132Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.19731871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54622024-02-06T22:18:03.133Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197361544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54632024-02-06T22:18:03.133Z{"msg":"[0]R flush_numbers[0..12]: [81, 91, 96, 94, 84, 91, 91, 91, 91, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197385481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54642024-02-06T22:18:03.133Z{"msg":"[0]R generation[0..12]: [39, 46, 49, 48, 42, 46, 46, 46, 46, 48, 46, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197401409Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54652024-02-06T22:18:03.133Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197423086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54662024-02-06T22:18:03.134Z{"msg":"[1]R flush_numbers[0..12]: [81, 91, 96, 94, 84, 91, 91, 91, 91, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197444553Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54672024-02-06T22:18:03.134Z{"msg":"[1]R generation[0..12]: [39, 46, 49, 48, 42, 46, 46, 46, 46, 48, 46, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.19746528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54682024-02-06T22:18:03.134Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197480628Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54692024-02-06T22:18:03.134Z{"msg":"Wait for a query_work_queue command to finish before sending IO
54702024-02-06T22:18:03.134Z[2]R flush_numbers[0..12]: [81, 91, 96, 94, 84, 91, 91, 91, 91, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197504955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54712024-02-06T22:18:03.135Z{"msg":"[2]R generation[0..12]: [39, 46, 49, 48, 42, 46, 46, 46, 46, 48, 46, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197520013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54722024-02-06T22:18:03.135Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.19754191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54732024-02-06T22:18:03.135Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197556908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54742024-02-06T22:18:03.136Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197577385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54752024-02-06T22:18:03.136Z{"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197597242Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54762024-02-06T22:18:03.192Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.19761163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54772024-02-06T22:18:03.192Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197633207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54782024-02-06T22:18:03.192Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197645656Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54792024-02-06T22:18:03.193Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197666213Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54802024-02-06T22:18:03.193Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197679851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54812024-02-06T22:18:03.193Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197694009Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54822024-02-06T22:18:03.193Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197707197Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54832024-02-06T22:18:03.193Z{"msg":"7a1ff404-cf30-4348-88d9-5bf91746d6a1 is now active with session: 62110184-d5a5-4f5a-9e7c-524c3d5b84b6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197727245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54842024-02-06T22:18:03.193Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.197740363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335,"session_id":"62110184-d5a5-4f5a-9e7c-524c3d5b84b6"}
54852024-02-06T22:18:03.193Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:52.19776041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335}
54862024-02-06T22:18:03.193ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54872024-02-06T22:18:03.193ZLoading write count information from file "/tmp/test_repair_verify.data"
54882024-02-06T22:18:03.193ZRun Repair workload
54892024-02-06T22:18:03.193ZError: Difference in extent metadata found!
54902024-02-06T22:18:03.193Z01/30 Write block 327 len 40960 data: 4 3 4 4 4 4 4 7 7 8
54912024-02-06T22:18:03.194Z02/30 Write block 40 len 32768 data: 5 4 4 4 4 3 4 5
54922024-02-06T22:18:03.194Z03/30 Write block 381 len 24576 data: 6 4 4 4 4 4
54932024-02-06T22:18:03.194Z04/30 Read block 21 len 8192
54942024-02-06T22:18:03.194Z05/30 Write block 420 len 40960 data: 6 6 4 4 5 5 6 5 6 5
54952024-02-06T22:18:03.194Z06/30 Read block 365 len 40960
54962024-02-06T22:18:03.194Z07/30 Read block 202 len 36864
54972024-02-06T22:18:03.194Z08/30 Read block 547 len 32768
54982024-02-06T22:18:03.194Z09/30 Write block 10 len 4096 data: 5
54992024-02-06T22:18:03.194Z10/30 Read block 523 len 24576
55002024-02-06T22:18:03.194Z11/30 Flush
55012024-02-06T22:18:03.194Z12/30 Write block 535 len 12288 data: 3 4 4
55022024-02-06T22:18:03.194Z13/30 Write block 133 len 8192 data: 4 4
55032024-02-06T22:18:03.194Z14/30 Read block 38 len 8192
55042024-02-06T22:18:03.195Z15/30 Write block 113 len 20480 data: 6 7 7 6 7
55052024-02-06T22:18:03.195Z16/30 Read block 388 len 16384
55062024-02-06T22:18:03.195Z17/30 Write block 410 len 24576 data: 5 6 3 3 4 5
55072024-02-06T22:18:03.195Z18/30 Read block 306 len 20480
55082024-02-06T22:18:03.195Z19/30 Read block 305 len 4096
55092024-02-06T22:18:03.195Z20/30 Flush
55102024-02-06T22:18:03.195Z21/30 Write block 587 len 40960 data: 4 4 6 8 8 7 9 8 9 7
55112024-02-06T22:18:03.195Z22/30 Read block 5 len 40960
55122024-02-06T22:18:03.195Z23/30 Read block 448 len 16384
55132024-02-06T22:18:03.195Z24/30 Write block 165 len 24576 data: 4 3 3 4 4 3
55142024-02-06T22:18:03.196Z25/30 Write block 574 len 28672 data: 7 7 8 9 8 6 7
55152024-02-06T22:18:03.196Z26/30 Read block 582 len 12288
55162024-02-06T22:18:03.196Z27/30 Write block 493 len 32768 data: 4 5 5 5 5 5 5 5
55172024-02-06T22:18:03.196Z28/30 Write block 59 len 32768 data: 7 8 6 4 3 4 4 5
55182024-02-06T22:18:03.196Z29/30 Write block 106 len 12288 data: 4 6 7
55192024-02-06T22:18:03.196Z30/30 Read block 484 len 32768
55202024-02-06T22:18:03.196Z----------------------------------------------------------------
55212024-02-06T22:18:03.196Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
55222024-02-06T22:18:03.196ZStates: Active Active Active
55232024-02-06T22:18:03.196ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
55242024-02-06T22:18:03.196Z 1 Acked 1000 Write 10 Done Done Done false
55252024-02-06T22:18:03.196Z 2 Acked 1001 Write 8 Done Done Done false
55262024-02-06T22:18:03.196Z 3 Acked 1002 Write 6 Done Done Done false
55272024-02-06T22:18:03.196Z 4 Acked 1003 Read 2 Done Done Done false
55282024-02-06T22:18:03.196Z 5 Acked 1004 Write 10 Done Done Done false
55292024-02-06T22:18:03.196Z 6 Acked 1005 Read 10 Done Done Done false
55302024-02-06T22:18:03.196Z 7 Acked 1006 Read 9 Done Done Done false
55312024-02-06T22:18:03.197Z 8 Acked 1007 Read 8 Done Done Done false
55322024-02-06T22:18:03.197Z 9 Acked 1008 Write 1 Done Done Done false
55332024-02-06T22:18:03.197Z 10 Acked 1009 Read 6 Done Done Done false
55342024-02-06T22:18:03.197Z 11 Acked 1010 Flush 0 Sent Done Done false
55352024-02-06T22:18:03.197Z 12 Acked 1011 Write 3 Sent Done Done false
55362024-02-06T22:18:03.197Z 13 Acked 1012 Write 2 Sent Done Done false
55372024-02-06T22:18:03.197Z 14 Acked 1013 Read 2 Sent Done Done false
55382024-02-06T22:18:03.197Z 15 Acked 1014 Write 5 Sent Done Done false
55392024-02-06T22:18:03.197Z 16 Acked 1015 Read 4 Sent Done Done false
55402024-02-06T22:18:03.197Z 17 Acked 1016 Write 6 Sent Done Done false
55412024-02-06T22:18:03.197Z 18 Acked 1017 Read 5 Sent Done Done false
55422024-02-06T22:18:03.197Z 19 Acked 1018 Read 1 Sent Done Done false
55432024-02-06T22:18:03.198Z 20 Acked 1019 Flush 0 Sent Done Done false
55442024-02-06T22:18:03.198Z 21 Acked 1020 Write 10 Sent Done Done false
55452024-02-06T22:18:03.198Z 22 Acked 1021 Read 10 Sent Done Done false
55462024-02-06T22:18:03.198Z 23 Acked 1022 Read 4 Sent Done Done false
55472024-02-06T22:18:03.198Z 24 Acked 1023 Write 6 Sent Done Done false
55482024-02-06T22:18:03.198Z 25 Acked 1024 Write 7 Sent Done Done false
55492024-02-06T22:18:03.198Z 26 Acked 1025 Read 3 Sent Done Done false
55502024-02-06T22:18:03.198Z 27 Acked 1026 Write 8 Sent Done Done false
55512024-02-06T22:18:03.198Z 28 Acked 1027 Write 8 Sent Done Done false
55522024-02-06T22:18:03.198Z 29 Acked 1028 Write 3 Sent Done Done false
55532024-02-06T22:18:03.198Z 30 Acked 1029 Read 8 Sent Sent Done false
55542024-02-06T22:18:03.198Z STATES DS:0 DS:1 DS:2 TOTAL
55552024-02-06T22:18:03.198Z New 0 0 0 0
55562024-02-06T22:18:03.198Z Sent 20 1 0 21
55572024-02-06T22:18:03.198Z Done 10 29 30 69
55582024-02-06T22:18:03.199Z Skipped 0 0 0 0
55592024-02-06T22:18:03.199Z Error 0 0 0 0
55602024-02-06T22:18:03.199ZLast Flush: 0 1019 1019
55612024-02-06T22:18:03.199ZDownstairs last five completed:
55622024-02-06T22:18:03.199ZUpstairs last five completed: 30 29 28 27 26
55632024-02-06T22:18:03.199Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:17:52.22766448Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1335}
55642024-02-06T22:18:03.199ZWrote out file "/tmp/test_repair_verify.data"
55652024-02-06T22:18:03.199Z
55662024-02-06T22:18:03.199ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55672024-02-06T22:18:03.199ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55682024-02-06T22:18:03.199Z 0 000-019 39 50 50 81 97 97 T F F <---
55692024-02-06T22:18:03.199Z 1 020-039 46 46 46 91 91 91 F F F
55702024-02-06T22:18:03.199Z 2 040-059 49 50 50 96 97 97 T T T <---
55712024-02-06T22:18:03.199Z 3 060-079 48 48 48 94 94 94 F T T <---
55722024-02-06T22:18:03.199Z 4 080-099 42 42 42 84 84 84 F F F
55732024-02-06T22:18:03.199Z 5 100-119 46 50 50 91 98 98 F T T <---
55742024-02-06T22:18:03.200Z 6 120-139 46 50 50 91 98 98 F F F <---
55752024-02-06T22:18:03.200Z 7 140-159 46 46 46 91 91 91 F F F
55762024-02-06T22:18:03.248Z 8 160-179 46 46 46 91 91 91 F T T <---
55772024-02-06T22:18:03.248Z 9 180-199 48 48 48 95 95 95 F F F
55782024-02-06T22:18:03.248Z 10 200-219 46 46 46 91 91 91 F F F
55792024-02-06T22:18:03.249Z 11 220-239 49 49 49 96 96 96 F F F
55802024-02-06T22:18:03.249Z 12 240-259 41 41 41 83 83 83 F F F
55812024-02-06T22:18:03.249Z 13 260-279 42 42 42 84 84 84 F F F
55822024-02-06T22:18:03.249Z 14 280-299 42 42 42 84 84 84 F F F
55832024-02-06T22:18:03.249Z 15 300-319 46 46 46 91 91 91 F F F
55842024-02-06T22:18:03.249Z 16 320-339 49 50 50 96 97 97 T F F <---
55852024-02-06T22:18:03.250Z 17 340-359 46 46 46 91 91 91 F F F
55862024-02-06T22:18:03.250Z 18 360-379 49 49 49 96 96 96 F F F
55872024-02-06T22:18:03.250Z 19 380-399 44 50 50 86 97 97 T F F <---
55882024-02-06T22:18:03.250Z 20 400-419 49 50 50 96 98 98 F F F <---
55892024-02-06T22:18:03.250Z 21 420-439 40 50 50 82 97 97 T F F <---
55902024-02-06T22:18:03.250Z 22 440-459 34 34 34 73 73 73 F F F
55912024-02-06T22:18:03.250Z 23 460-479 49 49 49 96 96 96 F F F
55922024-02-06T22:18:03.251Z 24 480-499 32 32 32 65 65 65 F T T <---
55932024-02-06T22:18:03.251Z 25 500-519 41 41 41 83 83 83 F T T <---
55942024-02-06T22:18:03.251Z 26 520-539 49 50 50 96 98 98 F F F <---
55952024-02-06T22:18:03.251Z 27 540-559 48 48 48 93 93 93 F F F
55962024-02-06T22:18:03.251Z 28 560-579 49 49 49 96 96 96 F T T <---
55972024-02-06T22:18:03.251Z 29 580-599 48 48 48 94 94 94 F T T <---
55982024-02-06T22:18:03.251ZMax gen: 50, Max flush: 98
55992024-02-06T22:18:03.251ZA Difference in extent metadata is expected here
56002024-02-06T22:18:03.251ZOn loop 25
56012024-02-06T22:18:03.251Z
56022024-02-06T22:18:03.252ZVerifying data now
56032024-02-06T22:18:03.252ZNo change after verify
56042024-02-06T22:18:03.252ZLoop: 25 Downstairs dump after verify (and repair):
56052024-02-06T22:18:03.252ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56062024-02-06T22:18:03.252Z 0 000-019 50 50 50 97 97 97 F F F
56072024-02-06T22:18:03.252Z 1 020-039 46 46 46 91 91 91 F F F
56082024-02-06T22:18:03.252Z 2 040-059 51 51 51 99 99 99 F F F
56092024-02-06T22:18:03.252Z 3 060-079 51 51 51 99 99 99 F F F
56102024-02-06T22:18:03.252Z 4 080-099 42 42 42 84 84 84 F F F
56112024-02-06T22:18:03.253Z 5 100-119 51 51 51 99 99 99 F F F
56122024-02-06T22:18:03.253Z 6 120-139 50 50 50 98 98 98 F F F
56132024-02-06T22:18:03.253Z 7 140-159 46 46 46 91 91 91 F F F
56142024-02-06T22:18:03.253Z 8 160-179 51 51 51 99 99 99 F F F
56152024-02-06T22:18:03.253Z 9 180-199 48 48 48 95 95 95 F F F
56162024-02-06T22:18:03.253Z 10 200-219 46 46 46 91 91 91 F F F
56172024-02-06T22:18:03.253Z 11 220-239 49 49 49 96 96 96 F F F
56182024-02-06T22:18:03.253Z 12 240-259 41 41 41 83 83 83 F F F
56192024-02-06T22:18:03.253Z 13 260-279 42 42 42 84 84 84 F F F
56202024-02-06T22:18:03.253Z 14 280-299 42 42 42 84 84 84 F F F
56212024-02-06T22:18:03.253Z 15 300-319 46 46 46 91 91 91 F F F
56222024-02-06T22:18:03.253Z 16 320-339 50 50 50 97 97 97 F F F
56232024-02-06T22:18:03.254Z 17 340-359 46 46 46 91 91 91 F F F
56242024-02-06T22:18:03.254Z 18 360-379 49 49 49 96 96 96 F F F
56252024-02-06T22:18:03.254Z 19 380-399 50 50 50 97 97 97 F F F
56262024-02-06T22:18:03.254Z 20 400-419 50 50 50 98 98 98 F F F
56272024-02-06T22:18:03.254Z 21 420-439 50 50 50 97 97 97 F F F
56282024-02-06T22:18:03.254Z 22 440-459 34 34 34 73 73 73 F F F
56292024-02-06T22:18:03.254Z 23 460-479 49 49 49 96 96 96 F F F
56302024-02-06T22:18:03.254Z 24 480-499 51 51 51 99 99 99 F F F
56312024-02-06T22:18:03.255Z 25 500-519 51 51 51 99 99 99 F F F
56322024-02-06T22:18:03.255Z 26 520-539 50 50 50 98 98 98 F F F
56332024-02-06T22:18:03.255Z 27 540-559 48 48 48 93 93 93 F F F
56342024-02-06T22:18:03.255Z 28 560-579 51 51 51 99 99 99 F F F
56352024-02-06T22:18:03.255Z 29 580-599 51 51 51 99 99 99 F F F
56362024-02-06T22:18:03.255ZMax gen: 51, Max flush: 99
56372024-02-06T22:18:03.255Z
56382024-02-06T22:18:03.256ZBegin loop 26
56392024-02-06T22:18:03.256ZDownstairs to restart: 2
56402024-02-06T22:18:03.256Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021241796Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343}
56412024-02-06T22:18:03.256Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021497842Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343}
56422024-02-06T22:18:03.256Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021529268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","Crucible runtime is spawned
56432024-02-06T22:18:03.256Zpid":1343}
56442024-02-06T22:18:03.256Z{"msg":"Crucible 734a9386-eb60-46e5-acfb-ee40acc31666 has session id: f597e8b6-c0ef-4073-8e05-810ddee7d82d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021567482Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56452024-02-06T22:18:03.257Z{"msg":"Upstairs opts: Upstairs UUID: 734a9386-eb60-46e5-acfb-ee40acc31666, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021611806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56462024-02-06T22:18:03.257Z{"msg":"Crucible stats registered with UUID: 734a9386-eb60-46e5-acfb-ee40acc31666","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021633503Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56472024-02-06T22:18:03.257Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021649751Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343}
56482024-02-06T22:18:03.257Z{"msg":"734a9386-eb60-46e5-acfb-ee40acc31666 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021664369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56492024-02-06T22:18:03.257Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021684887Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56502024-02-06T22:18:03.257Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021715612Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56512024-02-06T22:18:03.258Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021748068Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56522024-02-06T22:18:03.258Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021779134Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56532024-02-06T22:18:03.258Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021809919Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56542024-02-06T22:18:03.258Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021841985Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56552024-02-06T22:18:03.258Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021874871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56562024-02-06T22:18:03.258Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.021913305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56572024-02-06T22:18:03.259Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022002664Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56582024-02-06T22:18:03.259Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022038029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56592024-02-06T22:18:03.259Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022063325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56602024-02-06T22:18:03.259Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022087842Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56612024-02-06T22:18:03.260Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022746513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56622024-02-06T22:18:03.260Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022775689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56632024-02-06T22:18:03.260Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022792227Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56642024-02-06T22:18:03.260Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.022806855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56652024-02-06T22:18:03.260Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.023379258Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56662024-02-06T22:18:03.260Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.023419562Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56672024-02-06T22:18:03.260Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.02395462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56682024-02-06T22:18:03.260Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.023999684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56692024-02-06T22:18:03.261Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.02402356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56702024-02-06T22:18:03.261Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.024037418Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56712024-02-06T22:18:03.261Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.024051767Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56722024-02-06T22:18:03.261Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:17:58.024066245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56732024-02-06T22:18:03.261Z{Wait for a query_work_queue command to finish before sending IO
56742024-02-06T22:18:03.261Z"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054437312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56752024-02-06T22:18:03.262Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.05452397Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56762024-02-06T22:18:03.276Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054563774Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56772024-02-06T22:18:03.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.0545953Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56782024-02-06T22:18:03.276Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054613268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56792024-02-06T22:18:03.277Z{"msg":"[0]R flush_numbers[0..12]: [97, 91, 99, 99, 84, 99, 98, 91, 99, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054630845Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56802024-02-06T22:18:03.277Z{"msg":"[0]R generation[0..12]: [50, 46, 51, 51, 42, 51, 50, 46, 51, 48, 46, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054649883Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56812024-02-06T22:18:03.277Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.05466905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56822024-02-06T22:18:03.277Z{"msg":"[1]R flush_numbers[0..12]: [97, 91, 99, 99, 84, 99, 98, 91, 99, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054687967Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56832024-02-06T22:18:03.277Z{"msg":"[1]R generation[0..12]: [50, 46, 51, 51, 42, 51, 50, 46, 51, 48, 46, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054713924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56842024-02-06T22:18:03.278Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054732961Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56852024-02-06T22:18:03.278Z{"msg":"[2]R flush_numbers[0..12]: [97, 91, 99, 99, 84, 99, 98, 91, 99, 95, 91, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054751469Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56862024-02-06T22:18:03.278Z{"msg":"[2]R generation[0..12]: [50, 46, 51, 51, 42, 51, 50, 46, 51, 48, 46, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054768927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56872024-02-06T22:18:03.278Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054787154Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56882024-02-06T22:18:03.278Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054803852Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56892024-02-06T22:18:03.279Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054822609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56902024-02-06T22:18:03.279Z{"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054839697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56912024-02-06T22:18:03.279Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054856825Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56922024-02-06T22:18:03.279Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054873793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56932024-02-06T22:18:03.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.05488953Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56942024-02-06T22:18:03.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054906548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56952024-02-06T22:18:03.279Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054923616Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56962024-02-06T22:18:03.280Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054940464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56972024-02-06T22:18:03.280Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054956261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56982024-02-06T22:18:03.280Z{"msg":"734a9386-eb60-46e5-acfb-ee40acc31666 is now active with session: f597e8b6-c0ef-4073-8e05-810ddee7d82d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054972549Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
56992024-02-06T22:18:03.280Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.054990717Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343,"session_id":"f597e8b6-c0ef-4073-8e05-810ddee7d82d"}
57002024-02-06T22:18:03.280Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:02.055006524Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343}
57012024-02-06T22:18:03.280ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
57022024-02-06T22:18:03.280ZLoading write count information from file "/tmp/test_repair_verify.data"
57032024-02-06T22:18:03.281ZRun Repair workload
57042024-02-06T22:18:03.281Z01/30 Write block 342 len 40960 data: 7 5 5 5 6 7 7 6 8 10
57052024-02-06T22:18:03.281Z02/30 Read block 73 len 24576
57062024-02-06T22:18:03.281Z03/30 Write block 353 len 12288 data: 7 6 9
57072024-02-06T22:18:03.281Z04/30 Write block 173 len 20480 data: 6 5 4 4 6
57082024-02-06T22:18:03.281Z05/30 Flush
57092024-02-06T22:18:03.281Z06/30 Flush
57102024-02-06T22:18:03.282Z07/30 Flush
57112024-02-06T22:18:03.282Z08/30 Write block 40 len 20480 data: 6 5 5 5 5
57122024-02-06T22:18:03.282Z09/30 Write block 356 len 16384 data: 8 10 8 8
57132024-02-06T22:18:03.282Z10/30 Read block 542 len 16384
57142024-02-06T22:18:03.282Z11/30 Read block 526 len 8192
57152024-02-06T22:18:03.282Z12/30 Read block 313 len 8192
57162024-02-06T22:18:03.283Z13/30 Write block 416 len 24576 data: 6 8 8 8 7 7
57172024-02-06T22:18:03.283Z14/30 Read block 203 len 8192
57182024-02-06T22:18:03.283Z15/30 Read block 257 len 28672
57192024-02-06T22:18:03.283Z16/30 Read block 539 len 28672
57202024-02-06T22:18:03.283Z17/30 Read block 295 len 8192
57212024-02-06T22:18:03.283Z18/30 Read block 137 len 40960
57222024-02-06T22:18:03.283Z19/30 Read block 511 len 8192
57232024-02-06T22:18:03.284Z20/30 Flush
57242024-02-06T22:18:03.284Z21/30 Write block 278 len 16384 data: 5 6 6 6
57252024-02-06T22:18:03.284Z22/30 Flush
57262024-02-06T22:18:03.284Z23/30 Flush
57272024-02-06T22:18:03.284Z24/30 Write block 514 len 28672 data: 5 5 5 5 4 3 3
57282024-02-06T22:18:03.284Z25/30 Write block 218 len 28672 data: 4 4 6 6 5 5 5
57292024-02-06T22:18:03.284Z26/30 Write block 141 len 4096 data: 2
57302024-02-06T22:18:03.284Z27/30 Write block 381 len 16384 data: 7 5 5 5
57312024-02-06T22:18:03.284Z28/30 Write block 172 len 12288 data: 5 7 6
57322024-02-06T22:18:03.285Z29/30 Read block 422 len 36864
57332024-02-06T22:18:03.285Z30/30 Write block 248 len 32768 data: 6 7 6 5 5 5 7 5
57342024-02-06T22:18:03.285Z----------------------------------------------------------------
57352024-02-06T22:18:03.285Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57362024-02-06T22:18:03.285ZStates: Active Active Active
57372024-02-06T22:18:03.285ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57382024-02-06T22:18:03.285Z 1 Acked 1000 Write 10 Done Done Done false
57392024-02-06T22:18:03.286Z 2 Acked 1001 Read 6 Done Done Done false
57402024-02-06T22:18:03.286Z 3 Acked 1002 Write 3 Done Done Sent false
57412024-02-06T22:18:03.286Z 4 Acked 1003 Write 5 Done Done Sent false
57422024-02-06T22:18:03.287Z 5 Acked 1004 Flush 0 Done Done Sent false
57432024-02-06T22:18:03.287Z 6 Acked 1005 Flush 0 Done Done Sent false
57442024-02-06T22:18:03.287Z 7 Acked 1006 Flush 0 Done Done Sent false
57452024-02-06T22:18:03.288Z 8 Acked 1007 Write 5 Done Done Sent false
57462024-02-06T22:18:03.288Z 9 Acked 1008 Write 4 Done Done Sent false
57472024-02-06T22:18:03.288Z 10 Acked 1009 Read 4 Done Done Sent false
57482024-02-06T22:18:03.288Z 11 Acked 1010 Read 2 Done Done Sent false
57492024-02-06T22:18:03.288Z 12 Acked 1011 Read 2 Done Done Sent false
57502024-02-06T22:18:03.289Z 13 Acked 1012 Write 6 Done Done Sent false
57512024-02-06T22:18:03.289Z 14 Acked 1013 Read 2 Done Done Sent false
57522024-02-06T22:18:03.289Z 15 Acked 1014 Read 7 Done Done Sent false
57532024-02-06T22:18:03.289Z 16 Acked 1015 Read 7 Done Done Sent false
57542024-02-06T22:18:03.289Z 17 Acked 1016 Read 2 Done Done Sent false
57552024-02-06T22:18:03.289Z 18 Acked 1017 Read 10 Done Done Sent false
57562024-02-06T22:18:03.290Z 19 Acked 1018 Read 2 Done Done Sent false
57572024-02-06T22:18:03.290Z 20 Acked 1019 Flush 0 Done Done Sent false
57582024-02-06T22:18:03.290Z 21 Acked 1020 Write 4 Done Done Sent false
57592024-02-06T22:18:03.290Z 22 Acked 1021 Flush 0 Done Done Sent false
57602024-02-06T22:18:03.290Z 23 Acked 1022 Flush 0 Done Done Sent false
57612024-02-06T22:18:03.291Z 24 Acked 1023 Write 7 Done Done Sent false
57622024-02-06T22:18:03.291Z 25 Acked 1024 Write 7 Done Done Sent false
57632024-02-06T22:18:03.291Z 26 Acked 1025 Write 1 Done Done Sent false
57642024-02-06T22:18:03.291Z 27 Acked 1026 Write 4 Done Done Sent false
57652024-02-06T22:18:03.291Z 28 Acked 1027 Write 3 Done Done Sent false
57662024-02-06T22:18:03.292Z 29 Acked 1028 Read 9 Done Done Sent false
57672024-02-06T22:18:03.292Z 30 Acked 1029 Write 8 Sent Sent Sent false
57682024-02-06T22:18:03.292Z STATES DS:0 DS:1 DS:2 TOTAL
57692024-02-06T22:18:03.292Z New 0 0 0 0
57702024-02-06T22:18:03.292Z Sent 1 1 28 30
57712024-02-06T22:18:03.292Z Done 29 29 2 60
57722024-02-06T22:18:03.293Z Skipped 0 0 0 0
57732024-02-06T22:18:03.293Z Error 0 0 0 0
57742024-02-06T22:18:03.293ZLast Flush: 1022 1022 0
57752024-02-06T22:18:03.293ZDownstairs last five completed:
57762024-02-06T22:18:03.328ZUpstairs last five completed: 30 29 28 27 26
57772024-02-06T22:18:03.328Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:02.242666909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1343}
57782024-02-06T22:18:03.329ZWrote out file "/tmp/test_repair_verify.data"
57792024-02-06T22:18:03.329Z
57802024-02-06T22:18:03.329ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57812024-02-06T22:18:03.329ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57822024-02-06T22:18:03.329Z 0 000-019 50 50 50 97 97 97 F F F
57832024-02-06T22:18:03.329Z 1 020-039 46 46 46 91 91 91 F F F
57842024-02-06T22:18:03.330Z 2 040-059 52 52 51 103 103 99 F F F <---
57852024-02-06T22:18:03.330Z 3 060-079 51 51 51 99 99 99 F F F
57862024-02-06T22:18:03.330Z 4 080-099 42 42 42 84 84 84 F F F
57872024-02-06T22:18:03.330Z 5 100-119 51 51 51 99 99 99 F F F
57882024-02-06T22:18:03.330Z 6 120-139 50 50 50 98 98 98 F F F
57892024-02-06T22:18:03.330Z 7 140-159 46 46 46 91 91 91 T T F <---
57902024-02-06T22:18:03.330Z 8 160-179 52 52 51 100 100 99 T T F <---
57912024-02-06T22:18:03.331Z 9 180-199 48 48 48 95 95 95 F F F
57922024-02-06T22:18:03.331Z 10 200-219 46 46 46 91 91 91 T T F <---
57932024-02-06T22:18:03.331Z 11 220-239 49 49 49 96 96 96 T T F <---
57942024-02-06T22:18:03.331Z 12 240-259 41 41 41 83 83 83 T T F <---
57952024-02-06T22:18:03.331Z 13 260-279 52 52 42 104 104 84 F F F <---
57962024-02-06T22:18:03.331Z 14 280-299 52 52 42 104 104 84 F F F <---
57972024-02-06T22:18:03.331Z 15 300-319 46 46 46 91 91 91 F F F
57982024-02-06T22:18:03.331Z 16 320-339 50 50 50 97 97 97 F F F
57992024-02-06T22:18:03.332Z 17 340-359 52 52 46 103 103 91 F F T <---
58002024-02-06T22:18:03.332Z 18 360-379 49 49 49 96 96 96 F F F
58012024-02-06T22:18:03.332Z 19 380-399 50 50 50 97 97 97 T T F <---
58022024-02-06T22:18:03.332Z 20 400-419 52 52 50 103 103 98 F F F <---
58032024-02-06T22:18:03.332Z 21 420-439 52 52 50 103 103 97 F F F <---
58042024-02-06T22:18:03.332Z 22 440-459 34 34 34 73 73 73 F F F
58052024-02-06T22:18:03.332Z 23 460-479 49 49 49 96 96 96 F F F
58062024-02-06T22:18:03.333Z 24 480-499 51 51 51 99 99 99 F F F
58072024-02-06T22:18:03.333Z 25 500-519 51 51 51 99 99 99 T T F <---
58082024-02-06T22:18:03.333Z 26 520-539 50 50 50 98 98 98 T T F <---
58092024-02-06T22:18:03.333Z 27 540-559 48 48 48 93 93 93 F F F
58102024-02-06T22:18:03.333Z 28 560-579 51 51 51 99 99 99 F F F
58112024-02-06T22:18:03.334Z 29 580-599 51 51 51 99 99 99 F F F
58122024-02-06T22:18:03.334ZMax gen: 52, Max flush: 104
58132024-02-06T22:18:03.334ZError: Difference in extent metadata found!
58142024-02-06T22:18:03.334ZA Difference in extent metadata is expected here
58152024-02-06T22:18:03.334ZOn loop 26
58162024-02-06T22:18:03.334Z
58172024-02-06T22:18:03.334ZVerifying data now
58182024-02-06T22:18:08.754ZNo change after verify
58192024-02-06T22:18:08.754ZLoop: 26 Downstairs dump after verify (and repair):
58202024-02-06T22:18:08.766ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58212024-02-06T22:18:08.766Z 0 000-019 50 50 50 97 97 97 F F F
58222024-02-06T22:18:08.766Z 1 020-039 46 46 46 91 91 91 F F F
58232024-02-06T22:18:08.767Z 2 040-059 52 52 52 103 103 103 F F F
58242024-02-06T22:18:08.767Z 3 060-079 51 51 51 99 99 99 F F F
58252024-02-06T22:18:08.767Z 4 080-099 42 42 42 84 84 84 F F F
58262024-02-06T22:18:08.767Z 5 100-119 51 51 51 99 99 99 F F F
58272024-02-06T22:18:08.767Z 6 120-139 50 50 50 98 98 98 F F F
58282024-02-06T22:18:08.767Z 7 140-159 53 53 53 105 105 105 F F F
58292024-02-06T22:18:08.767Z 8 160-179 53 53 53 105 105 105 F F F
58302024-02-06T22:18:08.767Z 9 180-199 48 48 48 95 95 95 F F F
58312024-02-06T22:18:08.768Z 10 200-219 53 53 53 105 105 105 F F F
58322024-02-06T22:18:08.768Z 11 220-239 53 53 53 105 105 105 F F F
58332024-02-06T22:18:08.768Z 12 240-259 53 53 53 105 105 105 F F F
58342024-02-06T22:18:08.768Z 13 260-279 52 52 52 104 104 104 F F F
58352024-02-06T22:18:08.768Z 14 280-299 52 52 52 104 104 104 F F F
58362024-02-06T22:18:08.768Z 15 300-319 46 46 46 91 91 91 F F F
58372024-02-06T22:18:08.768Z 16 320-339 50 50 50 97 97 97 F F F
58382024-02-06T22:18:08.769Z 17 340-359 52 52 52 103 103 103 F F F
58392024-02-06T22:18:08.769Z 18 360-379 49 49 49 96 96 96 F F F
58402024-02-06T22:18:08.769Z 19 380-399 53 53 53 105 105 105 F F F
58412024-02-06T22:18:08.769Z 20 400-419 52 52 52 103 103 103 F F F
58422024-02-06T22:18:08.769Z 21 420-439 52 52 52 103 103 103 F F F
58432024-02-06T22:18:08.769Z 22 440-459 34 34 34 73 73 73 F F F
58442024-02-06T22:18:08.770Z 23 460-479 49 49 49 96 96 96 F F F
58452024-02-06T22:18:08.770Z 24 480-499 51 51 51 99 99 99 F F F
58462024-02-06T22:18:08.770Z 25 500-519 53 53 53 105 105 105 F F F
58472024-02-06T22:18:08.770Z 26 520-539 53 53 53 105 105 105 F F F
58482024-02-06T22:18:08.770Z 27 540-559 48 48 48 93 93 93 F F F
58492024-02-06T22:18:08.770Z 28 560-579 51 51 51 99 99 99 F F F
58502024-02-06T22:18:08.770Z 29 580-599 51 51 51 99 99 99 F F F
58512024-02-06T22:18:08.771ZMax gen: 53, Max flush: 105
58522024-02-06T22:18:09.183Z
58532024-02-06T22:18:09.183ZBegin loop 27
58542024-02-06T22:18:09.183ZDownstairs to restart: 0
58552024-02-06T22:18:09.191Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.006588764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351}
58562024-02-06T22:18:09.191Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.006904422Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351}
58572024-02-06T22:18:09.191ZCrucible runtime is spawned
58582024-02-06T22:18:09.192Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.006949335Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351}
58592024-02-06T22:18:09.192Z{"msg":"Crucible e636f7ee-524a-4816-afa3-480fc972d205 has session id: ed2b474b-593a-457c-8ef3-a528bf0f52cf","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.006983281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58602024-02-06T22:18:09.192Z{"msg":"Upstairs opts: Upstairs UUID: e636f7ee-524a-4816-afa3-480fc972d205, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007026185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58612024-02-06T22:18:09.192Z{"msg":"Crucible stats registered with UUID: e636f7ee-524a-4816-afa3-480fc972d205","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007054051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58622024-02-06T22:18:09.192Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007078478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351}
58632024-02-06T22:18:09.192Z{"msg":"e636f7ee-524a-4816-afa3-480fc972d205 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007097795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58642024-02-06T22:18:09.192Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007124842Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58652024-02-06T22:18:09.192Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.00713949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58662024-02-06T22:18:09.192Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007157807Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58672024-02-06T22:18:09.192Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007185653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58682024-02-06T22:18:09.192Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.00721185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58692024-02-06T22:18:09.192Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007237437Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58702024-02-06T22:18:09.192Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007300558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58712024-02-06T22:18:09.192Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007391346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58722024-02-06T22:18:09.193Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.00743621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58732024-02-06T22:18:09.193Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007465396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58742024-02-06T22:18:09.193Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007513899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58752024-02-06T22:18:09.193Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.007543255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58762024-02-06T22:18:09.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.008179759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58772024-02-06T22:18:09.193Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.008219704Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58782024-02-06T22:18:09.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.008236702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58792024-02-06T22:18:09.193Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.008280685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58802024-02-06T22:18:09.193Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.008848409Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58812024-02-06T22:18:09.193Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.008886884Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58822024-02-06T22:18:09.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.00943427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58832024-02-06T22:18:09.193Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.009471425Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58842024-02-06T22:18:09.193Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.009488262Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58852024-02-06T22:18:09.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.00950226Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58862024-02-06T22:18:09.193Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.009523048Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58872024-02-06T22:18:09.193Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.009537166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58882024-02-06T22:18:09.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.009767165Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58892024-02-06T22:18:09.193Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.009796431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58902024-02-06T22:18:09.193Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010287434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58912024-02-06T22:18:09.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010740063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58922024-02-06T22:18:09.193Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010783407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58932024-02-06T22:18:09.193Z{"msg":"[0]R flush_numbers[0..12]: [97, 91, 103, 99, 84, 99, 98, 105, 105, 95, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010804115Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58942024-02-06T22:18:09.193Z{"msg":"[0]R generation[0..12]: [50, 46, 52, 51, 42, 51, 50, 53, 53, 48, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010822442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58952024-02-06T22:18:09.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010850058Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58962024-02-06T22:18:09.194Z{"msg":"[1]R flush_numbers[0..12]: [97, 91, 103, 99, 84, 99, 98, 105, 105, 95, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010868836Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58972024-02-06T22:18:09.194Z{"msg":"[1]R generation[0..12]: [50, 46, 52, 51, 42, 51, 50, 53, 53, 48, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010894912Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58982024-02-06T22:18:09.194Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.01091325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
58992024-02-06T22:18:09.194Z{"msg":"[2]R flush_numbers[0..12]: [97, 91, 103, 99, 84, 99, 98, 105, 105, 95, 105, 105]","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO
59002024-02-06T22:18:09.194Z"2024-02-06T22:18:08.010939586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59012024-02-06T22:18:09.194Z{"msg":"[2]R generation[0..12]: [50, 46, 52, 51, 42, 51, 50, 53, 53, 48, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.010960893Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59022024-02-06T22:18:09.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.01098761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59032024-02-06T22:18:09.194Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011015216Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59042024-02-06T22:18:09.194Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011045922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59052024-02-06T22:18:09.194Z{"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011063029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59062024-02-06T22:18:09.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011080497Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59072024-02-06T22:18:09.194Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011097245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59082024-02-06T22:18:09.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011125311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59092024-02-06T22:18:09.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011170185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59102024-02-06T22:18:09.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011195112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59112024-02-06T22:18:09.194Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011221498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59122024-02-06T22:18:09.194Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011237646Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59132024-02-06T22:18:09.194Z{"msg":"e636f7ee-524a-4816-afa3-480fc972d205 is now active with session: ed2b474b-593a-457c-8ef3-a528bf0f52cf","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011262023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59142024-02-06T22:18:09.194Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.01127762Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351,"session_id":"ed2b474b-593a-457c-8ef3-a528bf0f52cf"}
59152024-02-06T22:18:09.194Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
59162024-02-06T22:18:09.194ZThe guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:08.011304787Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351}
59172024-02-06T22:18:09.194ZLoading write count information from file "/tmp/test_repair_verify.data"
59182024-02-06T22:18:09.194ZRun Repair workload
59192024-02-06T22:18:09.195Z01/30 Write block 460 len 36864 data: 4 4 3 3 3 3 2 5 6
59202024-02-06T22:18:09.195Z02/30 Read block 203 len 4096
59212024-02-06T22:18:09.195Z03/30 Write block 370 len 32768 data: 6 6 5 5 6 6 8 11
59222024-02-06T22:18:09.195Z04/30 Write block 583 len 32768 data: 3 3 4 3 5 5 7 9
59232024-02-06T22:18:09.195Z05/30 Read block 476 len 28672
59242024-02-06T22:18:09.195Z06/30 Read block 4 len 36864
59252024-02-06T22:18:09.195Z07/30 Write block 498 len 8192 data: 6 6
59262024-02-06T22:18:09.195Z08/30 Write block 398 len 8192 data: 4 4
59272024-02-06T22:18:09.195Z09/30 Flush
59282024-02-06T22:18:09.195Z10/30 Write block 2 len 16384 data: 3 4 4 4
59292024-02-06T22:18:09.195Z11/30 Write block 421 len 12288 data: 8 5 5
59302024-02-06T22:18:09.195Z12/30 Read block 325 len 32768
59312024-02-06T22:18:09.195Z13/30 Read block 50 len 4096
59322024-02-06T22:18:09.195Z14/30 Write block 316 len 12288 data: 6 5 4
59332024-02-06T22:18:09.195Z15/30 Write block 311 len 36864 data: 6 7 8 9 8 7 6 5 4
59342024-02-06T22:18:09.195Z16/30 Read block 126 len 4096
59352024-02-06T22:18:09.195Z17/30 Read block 96 len 32768
59362024-02-06T22:18:09.195Z18/30 Flush
59372024-02-06T22:18:09.195Z19/30 Write block 310 len 16384 data: 6 7 8 9
59382024-02-06T22:18:09.195Z20/30 Write block 305 len 8192 data: 5 5
59392024-02-06T22:18:09.195Z21/30 Write block 290 len 16384 data: 7 7 7 7
59402024-02-06T22:18:09.195Z22/30 Flush
59412024-02-06T22:18:09.195Z23/30 Write block 385 len 12288 data: 5 5 3
59422024-02-06T22:18:09.195Z24/30 Read block 515 len 28672
59432024-02-06T22:18:09.195Z25/30 Read block 25 len 32768
59442024-02-06T22:18:09.195Z26/30 Read block 563 len 16384
59452024-02-06T22:18:09.196Z27/30 Write block 39 len 32768 data: 4 7 6 6 6 6 4 5
59462024-02-06T22:18:09.196Z28/30 Write block 571 len 36864 data: 8 9 7 8 8 9 10 9 7
59472024-02-06T22:18:09.196Z29/30 Read block 426 len 20480
59482024-02-06T22:18:09.196Z30/30 Write block 526 len 12288 data: 5 5 5
59492024-02-06T22:18:09.196Z----------------------------------------------------------------
59502024-02-06T22:18:09.196Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59512024-02-06T22:18:09.196ZStates: Active Active Active
59522024-02-06T22:18:09.196ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59532024-02-06T22:18:09.196Z 1 Acked 1000 Write 9 Done Done Done false
59542024-02-06T22:18:09.196Z 2 Acked 1001 Read 1 Done Done Done false
59552024-02-06T22:18:09.202Z 3 Acked 1002 Write 8 Done Done Done false
59562024-02-06T22:18:09.202Z 4 Acked 1003 Write 8 Done Done Done false
59572024-02-06T22:18:09.202Z 5 Acked 1004 Read 7 Done Done Done false
59582024-02-06T22:18:09.202Z 6 Acked 1005 Read 9 Done Done Done false
59592024-02-06T22:18:09.202Z 7 Acked 1006 Write 2 Sent Done Done false
59602024-02-06T22:18:09.202Z 8 Acked 1007 Write 2 Sent Done Done false
59612024-02-06T22:18:09.203Z 9 Acked 1008 Flush 0 Sent Done Done false
59622024-02-06T22:18:09.203Z 10 Acked 1009 Write 4 Sent Done Done false
59632024-02-06T22:18:09.203Z 11 Acked 1010 Write 3 Sent Done Done false
59642024-02-06T22:18:09.203Z 12 Acked 1011 Read 8 Sent Done Done false
59652024-02-06T22:18:09.203Z 13 Acked 1012 Read 1 Sent Done Done false
59662024-02-06T22:18:09.203Z 14 Acked 1013 Write 3 Sent Done Done false
59672024-02-06T22:18:09.203Z 15 Acked 1014 Write 9 Sent Done Done false
59682024-02-06T22:18:09.203Z 16 Acked 1015 Read 1 Sent Done Done false
59692024-02-06T22:18:09.203Z 17 Acked 1016 Read 8 Sent Done Done false
59702024-02-06T22:18:09.203Z 18 Acked 1017 Flush 0 Sent Done Done false
59712024-02-06T22:18:09.203Z 19 Acked 1018 Write 4 Sent Done Done false
59722024-02-06T22:18:09.203Z 20 Acked 1019 Write 2 Sent Done Done false
59732024-02-06T22:18:09.203Z 21 Acked 1020 Write 4 Sent Done Done false
59742024-02-06T22:18:09.204Z 22 Acked 1021 Flush 0 Sent Done Done false
59752024-02-06T22:18:09.204Z 23 Acked 1022 Write 3 Sent Done Done false
59762024-02-06T22:18:09.204Z 24 Acked 1023 Read 7 Sent Done Done false
59772024-02-06T22:18:09.204Z 25 Acked 1024 Read 8 Sent Done Done false
59782024-02-06T22:18:09.204Z 26 Acked 1025 Read 4 Sent Done Done false
59792024-02-06T22:18:09.204Z 27 Acked 1026 Write 8 Sent Done Done false
59802024-02-06T22:18:09.204Z 28 Acked 1027 Write 9 Sent Done Done false
59812024-02-06T22:18:09.204Z 29 Acked 1028 Read 5 Sent Done Done false
59822024-02-06T22:18:09.204Z 30 Acked 1029 Write 3 Sent Sent Sent false
59832024-02-06T22:18:09.204Z STATES DS:0 DS:1 DS:2 TOTAL
59842024-02-06T22:18:09.204Z New 0 0 0 0
59852024-02-06T22:18:09.204Z Sent 24 1 1 26
59862024-02-06T22:18:09.204Z Done 6 29 29 64
59872024-02-06T22:18:09.205Z Skipped 0 0 0 0
59882024-02-06T22:18:09.205Z Error 0 0 0 0
59892024-02-06T22:18:09.205ZLast Flush: 0 1021 1021
59902024-02-06T22:18:09.205ZDownstairs last five completed:
59912024-02-06T22:18:09.205ZUpstairs last five completed: 30 29 28 27 26
59922024-02-06T22:18:09.205Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:08.041690211Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1351}
59932024-02-06T22:18:09.205ZWrote out file "/tmp/test_repair_verify.data"
59942024-02-06T22:18:09.205Z
59952024-02-06T22:18:09.205ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59962024-02-06T22:18:09.205ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59972024-02-06T22:18:09.205Z 0 000-019 50 54 54 97 107 107 F F F <---
59982024-02-06T22:18:09.205Z 1 020-039 46 46 46 91 91 91 F T T <---
59992024-02-06T22:18:09.205Z 2 040-059 52 52 52 103 103 103 F T T <---
60002024-02-06T22:18:09.205Z 3 060-079 51 51 51 99 99 99 F F F
60012024-02-06T22:18:09.206Z 4 080-099 42 42 42 84 84 84 F F F
60022024-02-06T22:18:09.206Z 5 100-119 51 51 51 99 99 99 F F F
60032024-02-06T22:18:09.206Z 6 120-139 50 50 50 98 98 98 F F F
60042024-02-06T22:18:09.206Z 7 140-159 53 53 53 105 105 105 F F F
60052024-02-06T22:18:09.206Z 8 160-179 53 53 53 105 105 105 F F F
60062024-02-06T22:18:09.206Z 9 180-199 48 48 48 95 95 95 F F F
60072024-02-06T22:18:09.206Z 10 200-219 53 53 53 105 105 105 F F F
60082024-02-06T22:18:09.206Z 11 220-239 53 53 53 105 105 105 F F F
60092024-02-06T22:18:09.206Z 12 240-259 53 53 53 105 105 105 F F F
60102024-02-06T22:18:09.206Z 13 260-279 52 52 52 104 104 104 F F F
60112024-02-06T22:18:09.206Z 14 280-299 52 54 54 104 108 108 F F F <---
60122024-02-06T22:18:09.206Z 15 300-319 46 54 54 91 108 108 F F F <---
60132024-02-06T22:18:09.206Z 16 320-339 50 50 50 97 97 97 F F F
60142024-02-06T22:18:09.207Z 17 340-359 52 52 52 103 103 103 F F F
60152024-02-06T22:18:09.207Z 18 360-379 49 54 54 96 106 106 T F F <---
60162024-02-06T22:18:09.207Z 19 380-399 53 54 54 105 106 106 F T T <---
60172024-02-06T22:18:09.207Z 20 400-419 52 52 52 103 103 103 F F F
60182024-02-06T22:18:09.207Z 21 420-439 52 54 54 103 107 107 F F F <---
60192024-02-06T22:18:09.207Z 22 440-459 34 34 34 73 73 73 F F F
60202024-02-06T22:18:09.207Z 23 460-479 49 54 54 96 106 106 T F F <---
60212024-02-06T22:18:09.207Z 24 480-499 51 54 54 99 106 106 F F F <---
60222024-02-06T22:18:09.207Z 25 500-519 53 53 53 105 105 105 F F F
60232024-02-06T22:18:09.207Z 26 520-539 53 53 53 105 105 105 F T T <---
60242024-02-06T22:18:09.207Z 27 540-559 48 48 48 93 93 93 F F F
60252024-02-06T22:18:09.207Z 28 560-579 51 51 51 99 99 99 F T T <---
60262024-02-06T22:18:09.208Z 29 580-599 51 54 54 99 106 106 T F F <---
60272024-02-06T22:18:09.208ZMax gen: 54, Max flush: 108
60282024-02-06T22:18:09.208ZError: Difference in extent metadata found!
60292024-02-06T22:18:09.208ZA Difference in extent metadata is expected here
60302024-02-06T22:18:09.208ZOn loop 27
60312024-02-06T22:18:09.208Z
60322024-02-06T22:18:09.208ZVerifying data now
60332024-02-06T22:18:14.486ZNo change after verify
60342024-02-06T22:18:14.486ZLoop: 27 Downstairs dump after verify (and repair):
60352024-02-06T22:18:14.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60362024-02-06T22:18:14.502Z 0 000-019 54 54 54 107 107 107 F F F
60372024-02-06T22:18:14.502Z 1 020-039 55 55 55 109 109 109 F F F
60382024-02-06T22:18:14.502Z 2 040-059 55 55 55 109 109 109 F F F
60392024-02-06T22:18:14.502Z 3 060-079 51 51 51 99 99 99 F F F
60402024-02-06T22:18:14.502Z 4 080-099 42 42 42 84 84 84 F F F
60412024-02-06T22:18:14.502Z 5 100-119 51 51 51 99 99 99 F F F
60422024-02-06T22:18:14.502Z 6 120-139 50 50 50 98 98 98 F F F
60432024-02-06T22:18:14.502Z 7 140-159 53 53 53 105 105 105 F F F
60442024-02-06T22:18:14.502Z 8 160-179 53 53 53 105 105 105 F F F
60452024-02-06T22:18:14.505Z 9 180-199 48 48 48 95 95 95 F F F
60462024-02-06T22:18:14.506Z 10 200-219 53 53 53 105 105 105 F F F
60472024-02-06T22:18:14.506Z 11 220-239 53 53 53 105 105 105 F F F
60482024-02-06T22:18:14.506Z 12 240-259 53 53 53 105 105 105 F F F
60492024-02-06T22:18:14.506Z 13 260-279 52 52 52 104 104 104 F F F
60502024-02-06T22:18:14.506Z 14 280-299 54 54 54 108 108 108 F F F
60512024-02-06T22:18:14.506Z 15 300-319 54 54 54 108 108 108 F F F
60522024-02-06T22:18:14.506Z 16 320-339 50 50 50 97 97 97 F F F
60532024-02-06T22:18:14.506Z 17 340-359 52 52 52 103 103 103 F F F
60542024-02-06T22:18:14.506Z 18 360-379 54 54 54 106 106 106 F F F
60552024-02-06T22:18:14.506Z 19 380-399 55 55 55 109 109 109 F F F
60562024-02-06T22:18:14.506Z 20 400-419 52 52 52 103 103 103 F F F
60572024-02-06T22:18:14.506Z 21 420-439 54 54 54 107 107 107 F F F
60582024-02-06T22:18:14.506Z 22 440-459 34 34 34 73 73 73 F F F
60592024-02-06T22:18:14.506Z 23 460-479 54 54 54 106 106 106 F F F
60602024-02-06T22:18:14.506Z 24 480-499 54 54 54 106 106 106 F F F
60612024-02-06T22:18:14.506Z 25 500-519 53 53 53 105 105 105 F F F
60622024-02-06T22:18:14.506Z 26 520-539 55 55 55 109 109 109 F F F
60632024-02-06T22:18:14.506Z 27 540-559 48 48 48 93 93 93 F F F
60642024-02-06T22:18:14.506Z 28 560-579 55 55 55 109 109 109 F F F
60652024-02-06T22:18:14.506Z 29 580-599 54 54 54 106 106 106 F F F
60662024-02-06T22:18:14.506ZMax gen: 55, Max flush: 109
60672024-02-06T22:18:14.506Z
60682024-02-06T22:18:14.506ZBegin loop 28
60692024-02-06T22:18:14.506ZDownstairs to restart: 2
60702024-02-06T22:18:14.524Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.747456912Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359}
60712024-02-06T22:18:14.524Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.747718017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359}
60722024-02-06T22:18:14.524Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.747745603Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359}
60732024-02-06T22:18:14.529ZCrucible runtime is spawned
60742024-02-06T22:18:14.529Z{"msg":"Crucible 53d28556-30fa-42c2-b959-604a18882b51 has session id: 4c6c9710-1671-4c66-af51-bfdcae93e244","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.747862787Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60752024-02-06T22:18:14.529Z{"msg":"Upstairs opts: Upstairs UUID: 53d28556-30fa-42c2-b959-604a18882b51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.747924929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60762024-02-06T22:18:14.530Z{"msg":"Crucible stats registered with UUID: 53d28556-30fa-42c2-b959-604a18882b51","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.747964083Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60772024-02-06T22:18:14.530Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748000998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359}
60782024-02-06T22:18:14.530Z{"msg":"53d28556-30fa-42c2-b959-604a18882b51 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748024995Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60792024-02-06T22:18:14.530Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748047462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60802024-02-06T22:18:14.530Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748074828Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60812024-02-06T22:18:14.530Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748116113Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60822024-02-06T22:18:14.530Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748143729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60832024-02-06T22:18:14.530Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748169076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60842024-02-06T22:18:14.530Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.74820935Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60852024-02-06T22:18:14.530Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748241416Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60862024-02-06T22:18:14.530Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748301878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60872024-02-06T22:18:14.530Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748372378Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60882024-02-06T22:18:14.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748427911Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60892024-02-06T22:18:14.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748473505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60902024-02-06T22:18:14.530Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.748497461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60912024-02-06T22:18:14.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.749064845Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60922024-02-06T22:18:14.530Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.749108369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60932024-02-06T22:18:14.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.749147603Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60942024-02-06T22:18:14.530Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.74917648Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60952024-02-06T22:18:14.530Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.749741273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60962024-02-06T22:18:14.530Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.749796726Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60972024-02-06T22:18:14.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750098155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60982024-02-06T22:18:14.530Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.75013375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
60992024-02-06T22:18:14.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750418462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61002024-02-06T22:18:14.531Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750472275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61012024-02-06T22:18:14.531Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750514059Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61022024-02-06T22:18:14.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750554224Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61032024-02-06T22:18:14.531Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750591448Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61042024-02-06T22:18:14.531Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750608026Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61052024-02-06T22:18:14.531Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.750709203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61062024-02-06T22:18:14.535Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751151453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61072024-02-06T22:18:14.535Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751192677Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61082024-02-06T22:18:14.535Z{"msg":"[0]R flush_numbers[0..12]: [107, 109, 109, 99, 84, 99, 98, 105, 105, 95, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751221513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61092024-02-06T22:18:14.535Z{"msg":"[0]R generation[0..12]: [54, 55, 55, 51, 42, 51, 50, 53, 53, 48, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751253649Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61102024-02-06T22:18:14.536Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751281145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61112024-02-06T22:18:14.536Z{"msg":"[1]R flush_numbers[0..12]: [107, 109, 109, 99, 84, 99, 98, 105, 105, 95, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.75132093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61122024-02-06T22:18:14.536Z{"msg":"[1]R generation[0..12]: [54, 55, 55, 51, 42, 51, 50, 53, 53, 48, 53, 53]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
61132024-02-06T22:18:14.536Z2024-02-06T22:18:13.751345726Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61142024-02-06T22:18:14.536Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751384931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61152024-02-06T22:18:14.536Z{"msg":"[2]R flush_numbers[0..12]: [107, 109, 109, 99, 84, 99, 98, 105, 105, 95, 105, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751419277Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61162024-02-06T22:18:14.536Z{"msg":"[2]R generation[0..12]: [54, 55, 55, 51, 42, 51, 50, 53, 53, 48, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751453312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61172024-02-06T22:18:14.536Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751493237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61182024-02-06T22:18:14.536Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751531232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61192024-02-06T22:18:14.536Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751566977Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61202024-02-06T22:18:14.536Z{"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751596673Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61212024-02-06T22:18:14.536Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751622109Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61222024-02-06T22:18:14.536Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751649145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61232024-02-06T22:18:14.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751680551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61242024-02-06T22:18:14.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751706168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61252024-02-06T22:18:14.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751742203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61262024-02-06T22:18:14.536Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751758321Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61272024-02-06T22:18:14.536Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751780878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61282024-02-06T22:18:14.536Z{"msg":"53d28556-30fa-42c2-b959-604a18882b51 is now active with session: 4c6c9710-1671-4c66-af51-bfdcae93e244","v":0,"nameRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61292024-02-06T22:18:14.536Z":"crucible","level":30,"time":"2024-02-06T22:18:13.751807984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61302024-02-06T22:18:14.536Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751830501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359,"session_id":"4c6c9710-1671-4c66-af51-bfdcae93e244"}
61312024-02-06T22:18:14.536Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:13.751852438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359}
61322024-02-06T22:18:14.536ZLoading write count information from file "/tmp/test_repair_verify.data"
61332024-02-06T22:18:14.536ZRun Repair workload
61342024-02-06T22:18:14.537Z01/30 Write block 436 len 16384 data: 6 6 5 5
61352024-02-06T22:18:14.537Z02/30 Read block 35 len 24576
61362024-02-06T22:18:14.537Z03/30 Read block 409 len 36864
61372024-02-06T22:18:14.537Z04/30 Write block 401 len 12288 data: 6 6 5
61382024-02-06T22:18:14.537Z05/30 Flush
61392024-02-06T22:18:14.541Z06/30 Write block 217 len 32768 data: 4 5 5 7 7 6 6 6
61402024-02-06T22:18:14.541Z07/30 Write block 253 len 12288 data: 6 8 6
61412024-02-06T22:18:14.541Z08/30 Read block 101 len 40960
61422024-02-06T22:18:14.541Z09/30 Write block 346 len 40960 data: 7 8 8 7 9 11 10 8 7 10
61432024-02-06T22:18:14.541Z10/30 Write block 220 len 28672 data: 8 8 7 7 7 6 6
61442024-02-06T22:18:14.541Z11/30 Write block 186 len 24576 data: 9 8 7 5 4 3
61452024-02-06T22:18:14.541Z12/30 Write block 192 len 8192 data: 3 3
61462024-02-06T22:18:14.541Z13/30 Read block 168 len 32768
61472024-02-06T22:18:14.544Z14/30 Read block 68 len 24576
61482024-02-06T22:18:14.544Z15/30 Flush
61492024-02-06T22:18:14.547Z16/30 Write block 294 len 12288 data: 7 8 6
61502024-02-06T22:18:14.547Z17/30 Write block 220 len 28672 data: 9 9 8 8 8 7 7
61512024-02-06T22:18:14.547Z18/30 Write block 279 len 16384 data: 7 7 7 6
61522024-02-06T22:18:14.547Z19/30 Write block 423 len 32768 data: 6 6 6 7 6 7 6 6
61532024-02-06T22:18:14.550Z20/30 Read block 147 len 4096
61542024-02-06T22:18:14.550Z21/30 Write block 587 len 32768 data: 6 6 8 10 9 8 10 9
61552024-02-06T22:18:14.550Z22/30 Write block 399 len 16384 data: 5 5 7 7
61562024-02-06T22:18:14.550Z23/30 Read block 473 len 36864
61572024-02-06T22:18:14.553Z24/30 Read block 84 len 16384
61582024-02-06T22:18:14.553Z25/30 Read block 40 len 36864
61592024-02-06T22:18:14.553Z26/30 Write block 112 len 12288 data: 5 7 8
61602024-02-06T22:18:14.556Z27/30 Read block 151 len 4096
61612024-02-06T22:18:14.556Z28/30 Write block 106 len 20480 data: 5 7 8 5 5
61622024-02-06T22:18:14.556Z29/30 Read block 269 len 24576
61632024-02-06T22:18:14.556Z30/30 Write block 54 len 24576 data: 13 13 11 9 8 8
61642024-02-06T22:18:14.560Z----------------------------------------------------------------
61652024-02-06T22:18:14.560Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61662024-02-06T22:18:14.560ZStates: Active Active Active
61672024-02-06T22:18:14.560ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61682024-02-06T22:18:14.560Z 1 Acked 1000 Write 4 Done Done Sent false
61692024-02-06T22:18:14.560Z 2 Acked 1001 Read 6 Done Done Sent false
61702024-02-06T22:18:14.560Z 3 Acked 1002 Read 9 Done Done Sent false
61712024-02-06T22:18:14.561Z 4 Acked 1003 Write 3 Done Done Sent false
61722024-02-06T22:18:14.561Z 5 Acked 1004 Flush 0 Done Done Sent false
61732024-02-06T22:18:14.561Z 6 Acked 1005 Write 8 Done Done Sent false
61742024-02-06T22:18:14.561Z 7 Acked 1006 Write 3 Done Done Sent false
61752024-02-06T22:18:14.561Z 8 Acked 1007 Read 10 Done Done Sent false
61762024-02-06T22:18:14.561Z 9 Acked 1008 Write 10 Done Done Sent false
61772024-02-06T22:18:14.561Z 10 Acked 1009 Write 7 Done Done Sent false
61782024-02-06T22:18:14.561Z 11 Acked 1010 Write 6 Done Done Sent false
61792024-02-06T22:18:14.561Z 12 Acked 1011 Write 2 Done Done Sent false
61802024-02-06T22:18:14.561Z 13 Acked 1012 Read 8 Done Done Sent false
61812024-02-06T22:18:14.561Z 14 Acked 1013 Read 6 Done Done Sent false
61822024-02-06T22:18:14.561Z 15 Acked 1014 Flush 0 Done Done Sent false
61832024-02-06T22:18:14.561Z 16 Acked 1015 Write 3 Done Done Sent false
61842024-02-06T22:18:14.561Z 17 Acked 1016 Write 7 Done Done Sent false
61852024-02-06T22:18:14.561Z 18 Acked 1017 Write 4 Done Done Sent false
61862024-02-06T22:18:14.561Z 19 Acked 1018 Write 8 Done Done Sent false
61872024-02-06T22:18:14.561Z 20 Acked 1019 Read 1 Done Done Sent false
61882024-02-06T22:18:14.561Z 21 Acked 1020 Write 8 Done Done Sent false
61892024-02-06T22:18:14.561Z 22 Acked 1021 Write 4 Done Done Sent false
61902024-02-06T22:18:14.561Z 23 Acked 1022 Read 9 Done Done Sent false
61912024-02-06T22:18:14.561Z 24 Acked 1023 Read 4 Done Done Sent false
61922024-02-06T22:18:14.561Z 25 Acked 1024 Read 9 Done Done Sent false
61932024-02-06T22:18:14.561Z 26 Acked 1025 Write 3 Done Done Sent false
61942024-02-06T22:18:14.561Z 27 Acked 1026 Read 1 Done Done Sent false
61952024-02-06T22:18:14.562Z 28 Acked 1027 Write 5 Done Done Sent false
61962024-02-06T22:18:14.562Z 29 Acked 1028 Read 6 Done Done Sent false
61972024-02-06T22:18:14.562Z 30 Acked 1029 Write 6 Sent Sent Sent false
61982024-02-06T22:18:14.562Z STATES DS:0 DS:1 DS:2 TOTAL
61992024-02-06T22:18:14.562Z New 0 0 0 0
62002024-02-06T22:18:14.562Z Sent 1 1 30 32
62012024-02-06T22:18:14.562Z Done 29 29 0 58
62022024-02-06T22:18:14.562Z Skipped 0 0 0 0
62032024-02-06T22:18:14.562Z Error 0 0 0 0
62042024-02-06T22:18:14.562ZLast Flush: 1014 1014 0
62052024-02-06T22:18:14.562ZDownstairs last five completed:
62062024-02-06T22:18:14.562ZUpstairs last five completed: 30 29 28 27 26
62072024-02-06T22:18:14.562Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:13.780799167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1359}
62082024-02-06T22:18:14.562ZWrote out file "/tmp/test_repair_verify.data"
62092024-02-06T22:18:14.569Z
62102024-02-06T22:18:14.569ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
62112024-02-06T22:18:14.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62122024-02-06T22:18:14.588Z 0 000-019 54 54 54 107 107 107 F F F
62132024-02-06T22:18:14.588Z 1 020-039 55 55 55 109 109 109 F F F
62142024-02-06T22:18:14.588Z 2 040-059 55 55 55 109 109 109 T T F <---
62152024-02-06T22:18:14.588Z 3 060-079 51 51 51 99 99 99 F F F
62162024-02-06T22:18:14.588Z 4 080-099 42 42 42 84 84 84 F F F
62172024-02-06T22:18:14.588Z 5 100-119 51 51 51 99 99 99 T T F <---
62182024-02-06T22:18:14.599Z 6 120-139 50 50 50 98 98 98 F F F
62192024-02-06T22:18:14.599Z 7 140-159 53 53 53 105 105 105 F F F
62202024-02-06T22:18:14.599Z 8 160-179 53 53 53 105 105 105 F F F
62212024-02-06T22:18:14.599Z 9 180-199 56 56 48 111 111 95 F F F <---
62222024-02-06T22:18:14.599Z 10 200-219 56 56 53 111 111 105 F F F <---
62232024-02-06T22:18:14.599Z 11 220-239 56 56 53 111 111 105 T T F <---
62242024-02-06T22:18:14.599Z 12 240-259 56 56 53 111 111 105 F F F <---
62252024-02-06T22:18:14.599Z 13 260-279 52 52 52 104 104 104 T T F <---
62262024-02-06T22:18:14.599Z 14 280-299 54 54 54 108 108 108 T T F <---
62272024-02-06T22:18:14.599Z 15 300-319 54 54 54 108 108 108 F F F
62282024-02-06T22:18:14.599Z 16 320-339 50 50 50 97 97 97 F F F
62292024-02-06T22:18:14.599Z 17 340-359 56 56 52 111 111 103 F F F <---
62302024-02-06T22:18:14.599Z 18 360-379 54 54 54 106 106 106 F F F
62312024-02-06T22:18:14.599Z 19 380-399 55 55 55 109 109 109 T T F <---
62322024-02-06T22:18:14.599Z 20 400-419 56 56 52 110 110 103 T T F <---
62332024-02-06T22:18:14.599Z 21 420-439 56 56 54 110 110 107 T T F <---
62342024-02-06T22:18:14.599Z 22 440-459 34 34 34 73 73 73 F F F
62352024-02-06T22:18:14.599Z 23 460-479 54 54 54 106 106 106 F F F
62362024-02-06T22:18:14.599Z 24 480-499 54 54 54 106 106 106 F F F
62372024-02-06T22:18:14.599Z 25 500-519 53 53 53 105 105 105 F F F
62382024-02-06T22:18:14.599Z 26 520-539 55 55 55 109 109 109 F F F
62392024-02-06T22:18:14.599Z 27 540-559 48 48 48 93 93 93 F F F
62402024-02-06T22:18:14.600Z 28 560-579 55 55 55 109 109 109 F F F
62412024-02-06T22:18:14.600Z 29 580-599 54 54 54 106 106 106 T T F <---
62422024-02-06T22:18:14.600ZMax gen: 56, Max flush: 111
62432024-02-06T22:18:14.600ZError: Difference in extent metadata found!
62442024-02-06T22:18:14.600ZA Difference in extent metadata is expected here
62452024-02-06T22:18:14.600ZOn loop 28
62462024-02-06T22:18:14.600Z
62472024-02-06T22:18:14.600ZVerifying data now
62482024-02-06T22:18:19.229ZNo change after verify
62492024-02-06T22:18:19.229ZLoop: 28 Downstairs dump after verify (and repair):
62502024-02-06T22:18:19.245ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62512024-02-06T22:18:19.245Z 0 000-019 54 54 54 107 107 107 F F F
62522024-02-06T22:18:19.245Z 1 020-039 55 55 55 109 109 109 F F F
62532024-02-06T22:18:19.245Z 2 040-059 57 57 57 112 112 112 F F F
62542024-02-06T22:18:19.245Z 3 060-079 51 51 51 99 99 99 F F F
62552024-02-06T22:18:19.245Z 4 080-099 42 42 42 84 84 84 F F F
62562024-02-06T22:18:19.245Z 5 100-119 57 57 57 112 112 112 F F F
62572024-02-06T22:18:19.245Z 6 120-139 50 50 50 98 98 98 F F F
62582024-02-06T22:18:19.245Z 7 140-159 53 53 53 105 105 105 F F F
62592024-02-06T22:18:19.245Z 8 160-179 53 53 53 105 105 105 F F F
62602024-02-06T22:18:19.245Z 9 180-199 56 56 56 111 111 111 F F F
62612024-02-06T22:18:19.245Z 10 200-219 56 56 56 111 111 111 F F F
62622024-02-06T22:18:19.245Z 11 220-239 57 57 57 112 112 112 F F F
62632024-02-06T22:18:19.245Z 12 240-259 56 56 56 111 111 111 F F F
62642024-02-06T22:18:19.245Z 13 260-279 57 57 57 112 112 112 F F F
62652024-02-06T22:18:19.249Z 14 280-299 57 57 57 112 112 112 F F F
62662024-02-06T22:18:19.249Z 15 300-319 54 54 54 108 108 108 F F F
62672024-02-06T22:18:19.249Z 16 320-339 50 50 50 97 97 97 F F F
62682024-02-06T22:18:19.249Z 17 340-359 56 56 56 111 111 111 F F F
62692024-02-06T22:18:19.249Z 18 360-379 54 54 54 106 106 106 F F F
62702024-02-06T22:18:19.249Z 19 380-399 57 57 57 112 112 112 F F F
62712024-02-06T22:18:19.249Z 20 400-419 57 57 57 112 112 112 F F F
62722024-02-06T22:18:19.249Z 21 420-439 57 57 57 112 112 112 F F F
62732024-02-06T22:18:19.249Z 22 440-459 34 34 34 73 73 73 F F F
62742024-02-06T22:18:19.249Z 23 460-479 54 54 54 106 106 106 F F F
62752024-02-06T22:18:19.249Z 24 480-499 54 54 54 106 106 106 F F F
62762024-02-06T22:18:19.249Z 25 500-519 53 53 53 105 105 105 F F F
62772024-02-06T22:18:19.249Z 26 520-539 55 55 55 109 109 109 F F F
62782024-02-06T22:18:19.249Z 27 540-559 48 48 48 93 93 93 F F F
62792024-02-06T22:18:19.249Z 28 560-579 55 55 55 109 109 109 F F F
62802024-02-06T22:18:19.249Z 29 580-599 57 57 57 112 112 112 F F F
62812024-02-06T22:18:19.249ZMax gen: 57, Max flush: 112
62822024-02-06T22:18:19.249Z
62832024-02-06T22:18:19.249ZBegin loop 29
62842024-02-06T22:18:19.249ZDownstairs to restart: 2
62852024-02-06T22:18:19.265Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487338557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367}
62862024-02-06T22:18:19.265Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",Crucible runtime is spawned
62872024-02-06T22:18:19.271Z\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487685181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367}
62882024-02-06T22:18:19.271Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487729624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367}
62892024-02-06T22:18:19.271Z{"msg":"Crucible 2c33a020-9382-470a-b855-b38694783758 has session id: 8b7a7655-a22d-42a9-80f7-6af0ae550a48","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.48776014Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62902024-02-06T22:18:19.271Z{"msg":"Upstairs opts: Upstairs UUID: 2c33a020-9382-470a-b855-b38694783758, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487820732Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62912024-02-06T22:18:19.271Z{"msg":"Crucible stats registered with UUID: 2c33a020-9382-470a-b855-b38694783758","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487863666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62922024-02-06T22:18:19.271Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487892413Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367}
62932024-02-06T22:18:19.271Z{"msg":"2c33a020-9382-470a-b855-b38694783758 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487923928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62942024-02-06T22:18:19.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487951834Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62952024-02-06T22:18:19.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.487987889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62962024-02-06T22:18:19.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488015676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62972024-02-06T22:18:19.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488043162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62982024-02-06T22:18:19.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488083317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
62992024-02-06T22:18:19.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488116112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63002024-02-06T22:18:19.272Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488226847Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63012024-02-06T22:18:19.272Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488302367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63022024-02-06T22:18:19.272Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.48835276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63032024-02-06T22:18:19.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488387415Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63042024-02-06T22:18:19.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488422091Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63052024-02-06T22:18:19.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.488470594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63062024-02-06T22:18:19.272Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.489004092Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63072024-02-06T22:18:19.272Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.489048936Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63082024-02-06T22:18:19.272Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.489078332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63092024-02-06T22:18:19.272Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.489116407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63102024-02-06T22:18:19.272Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.489555768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63112024-02-06T22:18:19.272Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.489714606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63122024-02-06T22:18:19.272Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490036983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63132024-02-06T22:18:19.272Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490082337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63142024-02-06T22:18:19.272Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490147988Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63152024-02-06T22:18:19.272Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490199691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63162024-02-06T22:18:19.272Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490226697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63172024-02-06T22:18:19.272Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490242915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63182024-02-06T22:18:19.272Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490257533Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63192024-02-06T22:18:19.273Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.49027895Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63202024-02-06T22:18:19.273Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.490753576Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63212024-02-06T22:18:19.273Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491201385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63222024-02-06T22:18:19.273Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.49124172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63232024-02-06T22:18:19.273Z{"msg":"[0]R flush_numbers[0..12]: [107, 109, 112, 99, 84, 112, 98, 105, 105, 111, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491258208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63242024-02-06T22:18:19.273Z{"msg":"[0]R generation[0..12]: [54, 55, 57, 51, 42, 57, 50, 53, 53, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491281624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63252024-02-06T22:18:19.273Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491296662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63262024-02-06T22:18:19.273Z{"msg":"[1]R flush_numbers[0..12]: [107, 109, 112, 99, 84, 112, 98, 105, 105, 111, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491320249Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63272024-02-06T22:18:19.273Z{"msg":"[1]R generation[0..12]: [54, 55, 57, 51, 42, 57, 50, 53, 53, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491335137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63282024-02-06T22:18:19.273Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491356754Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63292024-02-06T22:18:19.273Z{"msg":"[2]R flush_numbers[0..12]: [107, 109, 112, 99, 84, 112, 98, 105, 105, 111, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491371512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63302024-02-06T22:18:19.273ZWait for a query_work_queue command to finish before sending IO
63312024-02-06T22:18:19.273Z{"msg":"[2]R generation[0..12]: [54, 55, 57, 51, 42, 57, 50, 53, 53, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491396729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63322024-02-06T22:18:19.273Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491411937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63332024-02-06T22:18:19.273Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491432934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63342024-02-06T22:18:19.273Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491446922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63352024-02-06T22:18:19.273Z{"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.49146002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63362024-02-06T22:18:19.273Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491479898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63372024-02-06T22:18:19.273Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491493466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63382024-02-06T22:18:19.273Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491512333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63392024-02-06T22:18:19.273Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.49153879Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63402024-02-06T22:18:19.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491573885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63412024-02-06T22:18:19.277Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.49161299Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63422024-02-06T22:18:19.277Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491645905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63432024-02-06T22:18:19.277Z{"msg":"2c33a020-9382-470a-b855-b38694783758 is now active with session: 8b7a7655-a22d-42a9-80f7-6af0ae550a48","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491668662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63442024-02-06T22:18:19.277Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491696988Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367,"session_id":"8b7a7655-a22d-42a9-80f7-6af0ae550a48"}
63452024-02-06T22:18:19.277Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:18.491720725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367}
63462024-02-06T22:18:19.277ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63472024-02-06T22:18:19.277ZLoading write count information from file "/tmp/test_repair_verify.data"
63482024-02-06T22:18:19.277ZRun Repair workload
63492024-02-06T22:18:19.277Z01/30 Write block 318 len 20480 data: 6 5 4 4 4
63502024-02-06T22:18:19.277Z02/30 Read block 424 len 8192
63512024-02-06T22:18:19.277Z03/30 Read block 279 len 40960
63522024-02-06T22:18:19.277Z04/30 Write block 266 len 20480 data: 6 5 6 6 5
63532024-02-06T22:18:19.277Z05/30 Read block 355 len 24576
63542024-02-06T22:18:19.277Z06/30 Write block 501 len 40960 data: 5 6 7 5 5 6 6 5 4 4
63552024-02-06T22:18:19.277Z07/30 Write block 481 len 12288 data: 2 3 4
63562024-02-06T22:18:19.284Z08/30 Write block 73 len 36864 data: 4 6 5 2 3 3 3 4 4
63572024-02-06T22:18:19.284Z09/30 Write block 3 len 40960 data: 5 5 5 4 5 5 5 6 3 3
63582024-02-06T22:18:19.284Z10/30 Read block 475 len 20480
63592024-02-06T22:18:19.284Z11/30 Write block 533 len 16384 data: 5 5 4 5
63602024-02-06T22:18:19.284Z12/30 Write block 385 len 36864 data: 6 6 4 3 3 4 5 4 4
63612024-02-06T22:18:19.287Z13/30 Read block 471 len 32768
63622024-02-06T22:18:19.287Z14/30 Read block 437 len 20480
63632024-02-06T22:18:19.287Z15/30 Read block 238 len 20480
63642024-02-06T22:18:19.287Z16/30 Write block 443 len 12288 data: 4 3 4
63652024-02-06T22:18:19.287Z17/30 Read block 491 len 32768
63662024-02-06T22:18:19.287Z18/30 Read block 375 len 24576
63672024-02-06T22:18:19.291Z19/30 Write block 169 len 28672 data: 5 4 3 6 8 7 5
63682024-02-06T22:18:19.291Z20/30 Write block 468 len 20480 data: 7 6 6 7 7
63692024-02-06T22:18:19.291Z21/30 Write block 579 len 28672 data: 8 8 5 4 4 4 5
63702024-02-06T22:18:19.291Z22/30 Write block 350 len 36864 data: 10 12 11 9 8 11 9 11 9
63712024-02-06T22:18:19.291Z23/30 Read block 215 len 16384
63722024-02-06T22:18:19.294Z24/30 Write block 247 len 24576 data: 6 7 8 7 6 6
63732024-02-06T22:18:19.294Z25/30 Read block 66 len 12288
63742024-02-06T22:18:19.294Z26/30 Read block 451 len 28672
63752024-02-06T22:18:19.298Z27/30 Read block 380 len 12288
63762024-02-06T22:18:19.298Z28/30 Write block 409 len 40960 data: 5 6 7 4 4 5 6 7 9 9
63772024-02-06T22:18:19.298Z29/30 Read block 291 len 32768
63782024-02-06T22:18:19.302Z30/30 Read block 547 len 32768
63792024-02-06T22:18:19.302Z----------------------------------------------------------------
63802024-02-06T22:18:19.302Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63812024-02-06T22:18:19.302ZStates: Active Active Active
63822024-02-06T22:18:19.302ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63832024-02-06T22:18:19.302Z 1 Acked 1000 Write 5 Done Done Done false
63842024-02-06T22:18:19.302Z 2 Acked 1001 Read 2 Done Done Done false
63852024-02-06T22:18:19.302Z 3 Acked 1002 Read 10 Done Done Sent false
63862024-02-06T22:18:19.302Z 4 Acked 1003 Write 5 Done Done Sent false
63872024-02-06T22:18:19.302Z 5 Acked 1004 Read 6 Done Done Sent false
63882024-02-06T22:18:19.302Z 6 Acked 1005 Write 10 Done Done Sent false
63892024-02-06T22:18:19.302Z 7 Acked 1006 Write 3 Done Done Sent false
63902024-02-06T22:18:19.302Z 8 Acked 1007 Write 9 Done Done Sent false
63912024-02-06T22:18:19.302Z 9 Acked 1008 Write 10 Done Done Sent false
63922024-02-06T22:18:19.302Z 10 Acked 1009 Read 5 Done Done Sent false
63932024-02-06T22:18:19.302Z 11 Acked 1010 Write 4 Done Done Sent false
63942024-02-06T22:18:19.302Z 12 Acked 1011 Write 9 Done Done Sent false
63952024-02-06T22:18:19.302Z 13 Acked 1012 Read 8 Done Done Sent false
63962024-02-06T22:18:19.303Z 14 Acked 1013 Read 5 Done Done Sent false
63972024-02-06T22:18:19.303Z 15 Acked 1014 Read 5 Done Done Sent false
63982024-02-06T22:18:19.303Z 16 Acked 1015 Write 3 Done Done Sent false
63992024-02-06T22:18:19.303Z 17 Acked 1016 Read 8 Done Done Sent false
64002024-02-06T22:18:19.303Z 18 Acked 1017 Read 6 Done Done Sent false
64012024-02-06T22:18:19.303Z 19 Acked 1018 Write 7 Done Done Sent false
64022024-02-06T22:18:19.303Z 20 Acked 1019 Write 5 Done Done Sent false
64032024-02-06T22:18:19.303Z 21 Acked 1020 Write 7 Done Done Sent false
64042024-02-06T22:18:19.303Z 22 Acked 1021 Write 9 Done Done Sent false
64052024-02-06T22:18:19.303Z 23 Acked 1022 Read 4 Done Done Sent false
64062024-02-06T22:18:19.303Z 24 Acked 1023 Write 6 Done Done Sent false
64072024-02-06T22:18:19.303Z 25 Acked 1024 Read 3 Done Done Sent false
64082024-02-06T22:18:19.303Z 26 Acked 1025 Read 7 Done Done Sent false
64092024-02-06T22:18:19.303Z 27 Acked 1026 Read 3 Done Done Sent false
64102024-02-06T22:18:19.303Z 28 Acked 1027 Write 10 Done Done Sent false
64112024-02-06T22:18:19.303Z 29 Acked 1028 Read 8 Done Done Sent false
64122024-02-06T22:18:19.303Z 30 Acked 1029 Read 8 Done Done Sent false
64132024-02-06T22:18:19.303Z STATES DS:0 DS:1 DS:2 TOTAL
64142024-02-06T22:18:19.303Z New 0 0 0 0
64152024-02-06T22:18:19.303Z Sent 0 0 28 28
64162024-02-06T22:18:19.303Z Done 30 30 2 62
64172024-02-06T22:18:19.303Z Skipped 0 0 0 0
64182024-02-06T22:18:19.303Z Error 0 0 0 0
64192024-02-06T22:18:19.303ZLast Flush: 0 0 0
64202024-02-06T22:18:19.303ZDownstairs last five completed:
64212024-02-06T22:18:19.304ZUpstairs last five completed: 30 29 28 27 26
64222024-02-06T22:18:19.304Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:18.521604998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1367}
64232024-02-06T22:18:19.304ZWrote out file "/tmp/test_repair_verify.data"
64242024-02-06T22:18:19.307Z
64252024-02-06T22:18:19.307ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
64262024-02-06T22:18:19.327ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64272024-02-06T22:18:19.327Z 0 000-019 54 54 54 107 107 107 T T F <---
64282024-02-06T22:18:19.327Z 1 020-039 55 55 55 109 109 109 F F F
64292024-02-06T22:18:19.327Z 2 040-059 57 57 57 112 112 112 F F F
64302024-02-06T22:18:19.328Z 3 060-079 51 51 51 99 99 99 T T F <---
64312024-02-06T22:18:19.328Z 4 080-099 42 42 42 84 84 84 T T F <---
64322024-02-06T22:18:19.328Z 5 100-119 57 57 57 112 112 112 F F F
64332024-02-06T22:18:19.328Z 6 120-139 50 50 50 98 98 98 F F F
64342024-02-06T22:18:19.328Z 7 140-159 53 53 53 105 105 105 F F F
64352024-02-06T22:18:19.341Z 8 160-179 53 53 53 105 105 105 T T F <---
64362024-02-06T22:18:19.341Z 9 180-199 56 56 56 111 111 111 F F F
64372024-02-06T22:18:19.341Z 10 200-219 56 56 56 111 111 111 F F F
64382024-02-06T22:18:19.341Z 11 220-239 57 57 57 112 112 112 F F F
64392024-02-06T22:18:19.341Z 12 240-259 56 56 56 111 111 111 T T F <---
64402024-02-06T22:18:19.341Z 13 260-279 57 57 57 112 112 112 T T F <---
64412024-02-06T22:18:19.341Z 14 280-299 57 57 57 112 112 112 F F F
64422024-02-06T22:18:19.342Z 15 300-319 54 54 54 108 108 108 T T T
64432024-02-06T22:18:19.342Z 16 320-339 50 50 50 97 97 97 T T T
64442024-02-06T22:18:19.342Z 17 340-359 56 56 56 111 111 111 T T F <---
64452024-02-06T22:18:19.342Z 18 360-379 54 54 54 106 106 106 F F F
64462024-02-06T22:18:19.342Z 19 380-399 57 57 57 112 112 112 T T F <---
64472024-02-06T22:18:19.342Z 20 400-419 57 57 57 112 112 112 T T F <---
64482024-02-06T22:18:19.342Z 21 420-439 57 57 57 112 112 112 F F F
64492024-02-06T22:18:19.342Z 22 440-459 34 34 34 73 73 73 T T F <---
64502024-02-06T22:18:19.342Z 23 460-479 54 54 54 106 106 106 T T F <---
64512024-02-06T22:18:19.342Z 24 480-499 54 54 54 106 106 106 T T F <---
64522024-02-06T22:18:19.342Z 25 500-519 53 53 53 105 105 105 T T F <---
64532024-02-06T22:18:19.342Z 26 520-539 55 55 55 109 109 109 T T F <---
64542024-02-06T22:18:19.342Z 27 540-559 48 48 48 93 93 93 F F F
64552024-02-06T22:18:19.342Z 28 560-579 55 55 55 109 109 109 T T F <---
64562024-02-06T22:18:19.342Z 29 580-599 57 57 57 112 112 112 T T F <---
64572024-02-06T22:18:19.342ZMax gen: 57, Max flush: 112
64582024-02-06T22:18:19.342ZError: Difference in extent metadata found!
64592024-02-06T22:18:19.342ZA Difference in extent metadata is expected here
64602024-02-06T22:18:19.342ZOn loop 29
64612024-02-06T22:18:19.342Z
64622024-02-06T22:18:19.347ZVerifying data now
64632024-02-06T22:18:25.207ZNo change after verify
64642024-02-06T22:18:25.207ZLoop: 29 Downstairs dump after verify (and repair):
64652024-02-06T22:18:25.221ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64662024-02-06T22:18:25.221Z 0 000-019 58 58 58 113 113 113 F F F
64672024-02-06T22:18:25.221Z 1 020-039 55 55 55 109 109 109 F F F
64682024-02-06T22:18:25.221Z 2 040-059 57 57 57 112 112 112 F F F
64692024-02-06T22:18:25.221Z 3 060-079 58 58 58 113 113 113 F F F
64702024-02-06T22:18:25.222Z 4 080-099 58 58 58 113 113 113 F F F
64712024-02-06T22:18:25.222Z 5 100-119 57 57 57 112 112 112 F F F
64722024-02-06T22:18:25.222Z 6 120-139 50 50 50 98 98 98 F F F
64732024-02-06T22:18:25.222Z 7 140-159 53 53 53 105 105 105 F F F
64742024-02-06T22:18:25.222Z 8 160-179 58 58 58 113 113 113 F F F
64752024-02-06T22:18:25.222Z 9 180-199 56 56 56 111 111 111 F F F
64762024-02-06T22:18:25.222Z 10 200-219 56 56 56 111 111 111 F F F
64772024-02-06T22:18:25.222Z 11 220-239 57 57 57 112 112 112 F F F
64782024-02-06T22:18:25.222Z 12 240-259 58 58 58 113 113 113 F F F
64792024-02-06T22:18:25.222Z 13 260-279 58 58 58 113 113 113 F F F
64802024-02-06T22:18:25.222Z 14 280-299 57 57 57 112 112 112 F F F
64812024-02-06T22:18:25.236Z 15 300-319 58 58 58 113 113 113 F F F
64822024-02-06T22:18:25.236Z 16 320-339 58 58 58 113 113 113 F F F
64832024-02-06T22:18:25.236Z 17 340-359 58 58 58 113 113 113 F F F
64842024-02-06T22:18:25.236Z 18 360-379 54 54 54 106 106 106 F F F
64852024-02-06T22:18:25.236Z 19 380-399 58 58 58 113 113 113 F F F
64862024-02-06T22:18:25.236Z 20 400-419 58 58 58 113 113 113 F F F
64872024-02-06T22:18:25.237Z 21 420-439 57 57 57 112 112 112 F F F
64882024-02-06T22:18:25.237Z 22 440-459 58 58 58 113 113 113 F F F
64892024-02-06T22:18:25.237Z 23 460-479 58 58 58 113 113 113 F F F
64902024-02-06T22:18:25.237Z 24 480-499 58 58 58 113 113 113 F F F
64912024-02-06T22:18:25.237Z 25 500-519 58 58 58 113 113 113 F F F
64922024-02-06T22:18:25.237Z 26 520-539 58 58 58 113 113 113 F F F
64932024-02-06T22:18:25.237Z 27 540-559 48 48 48 93 93 93 F F F
64942024-02-06T22:18:25.237Z 28 560-579 58 58 58 113 113 113 F F F
64952024-02-06T22:18:25.237Z 29 580-599 58 58 58 113 113 113 F F F
64962024-02-06T22:18:25.237ZMax gen: 58, Max flush: 113
64972024-02-06T22:18:25.238Z
64982024-02-06T22:18:25.238ZBegin loop 30
64992024-02-06T22:18:25.238ZDownstairs to restart: 1
65002024-02-06T22:18:25.251Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463215594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375}
65012024-02-06T22:18:25.251Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463478748Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375}
65022024-02-06T22:18:25.251ZCrucible runtime is spawned
65032024-02-06T22:18:25.251Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463599672Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375}
65042024-02-06T22:18:25.251Z{"msg":"Crucible 60edf462-83c5-4971-90f2-e1b1b917e0fd has session id: 8c2b0059-69bc-4ce7-a4a0-949c9da29d03","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463649525Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65052024-02-06T22:18:25.251Z{"msg":"Upstairs opts: Upstairs UUID: 60edf462-83c5-4971-90f2-e1b1b917e0fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463695639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65062024-02-06T22:18:25.251Z{"msg":"Crucible stats registered with UUID: 60edf462-83c5-4971-90f2-e1b1b917e0fd","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463715256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65072024-02-06T22:18:25.252Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463736143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375}
65082024-02-06T22:18:25.252Z{"msg":"60edf462-83c5-4971-90f2-e1b1b917e0fd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463749922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65092024-02-06T22:18:25.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463781947Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65102024-02-06T22:18:25.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463829831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65112024-02-06T22:18:25.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463858047Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65122024-02-06T22:18:25.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463915399Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65132024-02-06T22:18:25.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.463986199Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65142024-02-06T22:18:25.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464019785Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65152024-02-06T22:18:25.252Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464043652Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65162024-02-06T22:18:25.252Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464097244Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65172024-02-06T22:18:25.252Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464140519Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65182024-02-06T22:18:25.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464183543Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65192024-02-06T22:18:25.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464211249Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65202024-02-06T22:18:25.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464249114Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65212024-02-06T22:18:25.252Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464845813Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65222024-02-06T22:18:25.252Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464891817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65232024-02-06T22:18:25.252Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464926163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65242024-02-06T22:18:25.252Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.464995983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65252024-02-06T22:18:25.252Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.46553507Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65262024-02-06T22:18:25.252Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.465591463Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65272024-02-06T22:18:25.252Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.466166585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65282024-02-06T22:18:25.252Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.466226507Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65292024-02-06T22:18:25.252Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.4662772Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65302024-02-06T22:18:25.252Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.466315715Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65312024-02-06T22:18:25.253Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.46635379Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65322024-02-06T22:18:25.253Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.466387605Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65332024-02-06T22:18:25.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.466419191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65342024-02-06T22:18:25.253Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.466452666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65352024-02-06T22:18:25.253Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.466778242Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65362024-02-06T22:18:25.253Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467209094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65372024-02-06T22:18:25.253Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467256678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65382024-02-06T22:18:25.253Z{"msg":"[0]R flush_numbers[0..12]: [113, 109, 112, 113, 113, 112, 98, 105, 113, 111, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467275465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65392024-02-06T22:18:25.253Z{"msg":"[0]R generation[0..12]: [58, 55, 57, 58, 58, 57, 50, 53, 58, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467299132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65402024-02-06T22:18:25.253Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.46731457Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65412024-02-06T22:18:25.253Z{"msg":"[1]R flush_numbers[0..12]: [113, 109, 112, 113, 113, 112, 98, 105, 113, 111, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467337577Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65422024-02-06T22:18:25.253Z{"msg":"[1]R generation[0..12]: [58, 55, 57, 58, 58, 57, 50, 53, 58, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467352615Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65432024-02-06T22:18:25.253Z{Wait for a query_work_queue command to finish before sending IO
65442024-02-06T22:18:25.253Z"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467371222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65452024-02-06T22:18:25.253Z{"msg":"[2]R flush_numbers[0..12]: [113, 109, 112, 113, 113, 112, 98, 105, 113, 111, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.46738607Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65462024-02-06T22:18:25.253Z{"msg":"[2]R generation[0..12]: [58, 55, 57, 58, 58, 57, 50, 53, 58, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467409317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65472024-02-06T22:18:25.253Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467424465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65482024-02-06T22:18:25.253Z{"msg":"Max found gen is 59","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467450691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65492024-02-06T22:18:25.253Z{"msg":"Generation requested: 60 >= found:59","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.46746504Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65502024-02-06T22:18:25.253Z{"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467478028Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65512024-02-06T22:18:25.253Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467498475Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65522024-02-06T22:18:25.253Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467512303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65532024-02-06T22:18:25.253Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467524361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65542024-02-06T22:18:25.253Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467544959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65552024-02-06T22:18:25.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467558737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65562024-02-06T22:18:25.254Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467571995Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65572024-02-06T22:18:25.254Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467597491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65582024-02-06T22:18:25.254Z{"msg":"60edf462-83c5-4971-90f2-e1b1b917e0fd is now active with session: 8c2b0059-69bc-4ce7-a4a0-949c9da29d03","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.46761114Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65592024-02-06T22:18:25.254Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467630217Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375,"session_id":"8c2b0059-69bc-4ce7-a4a0-949c9da29d03"}
65602024-02-06T22:18:25.254Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:24.467642945Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375}
65612024-02-06T22:18:25.254ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65622024-02-06T22:18:25.254ZLoading write count information from file "/tmp/test_repair_verify.data"
65632024-02-06T22:18:25.254ZRun Repair workload
65642024-02-06T22:18:25.254Z01/30 Write block 529 len 20480 data: 5 6 6 5 6
65652024-02-06T22:18:25.258Z02/30 Write block 247 len 16384 data: 7 8 9 8
65662024-02-06T22:18:25.258Z03/30 Write block 358 len 28672 data: 10 9 8 8 10 10 8
65672024-02-06T22:18:25.258Z04/30 Write block 144 len 16384 data: 6 7 8 9
65682024-02-06T22:18:25.258Z05/30 Read block 28 len 32768
65692024-02-06T22:18:25.258Z06/30 Flush
65702024-02-06T22:18:25.258Z07/30 Write block 397 len 16384 data: 4 5 6 6
65712024-02-06T22:18:25.258Z08/30 Flush
65722024-02-06T22:18:25.258Z09/30 Write block 90 len 8192 data: 3 3
65732024-02-06T22:18:25.258Z10/30 Write block 219 len 24576 data: 6 10 10 9 9 9
65742024-02-06T22:18:25.258Z11/30 Read block 280 len 36864
65752024-02-06T22:18:25.262Z12/30 Write block 178 len 40960 data: 6 8 8 8 7 8 10 10 10 9
65762024-02-06T22:18:25.262Z13/30 Read block 568 len 16384
65772024-02-06T22:18:25.262Z14/30 Write block 348 len 40960 data: 9 8 11 13 12 10 9 12 10 12
65782024-02-06T22:18:25.262Z15/30 Read block 397 len 36864
65792024-02-06T22:18:25.265Z16/30 Read block 572 len 20480
65802024-02-06T22:18:25.265Z17/30 Write block 18 len 16384 data: 4 5 4 4
65812024-02-06T22:18:25.265Z18/30 Read block 132 len 24576
65822024-02-06T22:18:25.265Z19/30 Write block 56 len 8192 data: 12 10
65832024-02-06T22:18:25.269Z20/30 Write block 230 len 32768 data: 3 5 5 3 4 4 4 4
65842024-02-06T22:18:25.269Z21/30 Read block 457 len 32768
65852024-02-06T22:18:25.269Z22/30 Read block 345 len 32768
65862024-02-06T22:18:25.272Z23/30 Write block 525 len 36864 data: 4 6 6 6 6 7 7 6 7
65872024-02-06T22:18:25.272Z24/30 Read block 131 len 24576
65882024-02-06T22:18:25.272Z25/30 Read block 434 len 36864
65892024-02-06T22:18:25.275Z26/30 Read block 183 len 40960
65902024-02-06T22:18:25.275Z27/30 Write block 179 len 36864 data: 9 9 9 8 9 11 11 11 10
65912024-02-06T22:18:25.275Z28/30 Read block 343 len 36864
65922024-02-06T22:18:25.279Z29/30 Read block 119 len 8192
65932024-02-06T22:18:25.280Z30/30 Read block 441 len 28672
65942024-02-06T22:18:25.280Z----------------------------------------------------------------
65952024-02-06T22:18:25.280Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65962024-02-06T22:18:25.280ZStates: Active Active Active
65972024-02-06T22:18:25.280ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65982024-02-06T22:18:25.280Z 1 Acked 1000 Write 5 Done Done Done false
65992024-02-06T22:18:25.280Z 2 Acked 1001 Write 4 Done Done Done false
66002024-02-06T22:18:25.280Z 3 Acked 1002 Write 7 Done Done Done false
66012024-02-06T22:18:25.280Z 4 Acked 1003 Write 4 Done Sent Done false
66022024-02-06T22:18:25.280Z 5 Acked 1004 Read 8 Done Sent Done false
66032024-02-06T22:18:25.280Z 6 Acked 1005 Flush 0 Done Sent Done false
66042024-02-06T22:18:25.280Z 7 Acked 1006 Write 4 Done Sent Done false
66052024-02-06T22:18:25.280Z 8 Acked 1007 Flush 0 Done Sent Done false
66062024-02-06T22:18:25.280Z 9 Acked 1008 Write 2 Done Sent Done false
66072024-02-06T22:18:25.280Z 10 Acked 1009 Write 6 Done Sent Done false
66082024-02-06T22:18:25.280Z 11 Acked 1010 Read 9 Done Sent Done false
66092024-02-06T22:18:25.280Z 12 Acked 1011 Write 10 Done Sent Done false
66102024-02-06T22:18:25.281Z 13 Acked 1012 Read 4 Done Sent Done false
66112024-02-06T22:18:25.281Z 14 Acked 1013 Write 10 Done Sent Done false
66122024-02-06T22:18:25.281Z 15 Acked 1014 Read 9 Done Sent Done false
66132024-02-06T22:18:25.281Z 16 Acked 1015 Read 5 Done Sent Done false
66142024-02-06T22:18:25.281Z 17 Acked 1016 Write 4 Done Sent Done false
66152024-02-06T22:18:25.281Z 18 Acked 1017 Read 6 Done Sent Done false
66162024-02-06T22:18:25.281Z 19 Acked 1018 Write 2 Done Sent Done false
66172024-02-06T22:18:25.281Z 20 Acked 1019 Write 8 Done Sent Done false
66182024-02-06T22:18:25.281Z 21 Acked 1020 Read 8 Done Sent Done false
66192024-02-06T22:18:25.281Z 22 Acked 1021 Read 8 Done Sent Done false
66202024-02-06T22:18:25.281Z 23 Acked 1022 Write 9 Done Sent Done false
66212024-02-06T22:18:25.281Z 24 Acked 1023 Read 6 Done Sent Done false
66222024-02-06T22:18:25.281Z 25 Acked 1024 Read 9 Done Sent Done false
66232024-02-06T22:18:25.281Z 26 Acked 1025 Read 10 Done Sent Done false
66242024-02-06T22:18:25.281Z 27 Acked 1026 Write 9 Done Sent Done false
66252024-02-06T22:18:25.281Z 28 Acked 1027 Read 9 Done Sent Done false
66262024-02-06T22:18:25.281Z 29 Acked 1028 Read 2 Done Sent Done false
66272024-02-06T22:18:25.281Z 30 Acked 1029 Read 7 Done Sent Done false
66282024-02-06T22:18:25.281Z STATES DS:0 DS:1 DS:2 TOTAL
66292024-02-06T22:18:25.281Z New 0 0 0 0
66302024-02-06T22:18:25.281Z Sent 0 27 0 27
66312024-02-06T22:18:25.281Z Done 30 3 30 63
66322024-02-06T22:18:25.281Z Skipped 0 0 0 0
66332024-02-06T22:18:25.281Z Error 0 0 0 0
66342024-02-06T22:18:25.281ZLast Flush: 1007 0 1007
66352024-02-06T22:18:25.282ZDownstairs last five completed:
66362024-02-06T22:18:25.282ZUpstairs last five completed: 30 29 28 27 26
66372024-02-06T22:18:25.282Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:24.499544115Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1375}
66382024-02-06T22:18:25.282ZWrote out file "/tmp/test_repair_verify.data"
66392024-02-06T22:18:25.286Z
66402024-02-06T22:18:25.286ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66412024-02-06T22:18:25.304ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66422024-02-06T22:18:25.304Z 0 000-019 58 58 58 113 113 113 T F T <---
66432024-02-06T22:18:25.304Z 1 020-039 55 55 55 109 109 109 T F T <---
66442024-02-06T22:18:25.304Z 2 040-059 57 57 57 112 112 112 T F T <---
66452024-02-06T22:18:25.304Z 3 060-079 58 58 58 113 113 113 F F F
66462024-02-06T22:18:25.304Z 4 080-099 58 58 58 113 113 113 T F T <---
66472024-02-06T22:18:25.304Z 5 100-119 57 57 57 112 112 112 F F F
66482024-02-06T22:18:25.304Z 6 120-139 50 50 50 98 98 98 F F F
66492024-02-06T22:18:25.304Z 7 140-159 60 53 60 114 105 114 F F F <---
66502024-02-06T22:18:25.304Z 8 160-179 58 58 58 113 113 113 T F T <---
66512024-02-06T22:18:25.304Z 9 180-199 56 56 56 111 111 111 T F T <---
66522024-02-06T22:18:25.307Z 10 200-219 56 56 56 111 111 111 T F T <---
66532024-02-06T22:18:25.308Z 11 220-239 57 57 57 112 112 112 T F T <---
66542024-02-06T22:18:25.308Z 12 240-259 60 58 60 114 113 114 F T F <---
66552024-02-06T22:18:25.308Z 13 260-279 58 58 58 113 113 113 F F F
66562024-02-06T22:18:25.308Z 14 280-299 57 57 57 112 112 112 F F F
66572024-02-06T22:18:25.308Z 15 300-319 58 58 58 113 113 113 F F F
66582024-02-06T22:18:25.308Z 16 320-339 58 58 58 113 113 113 F F F
66592024-02-06T22:18:25.308Z 17 340-359 60 58 60 114 113 114 T T T <---
66602024-02-06T22:18:25.308Z 18 360-379 60 54 60 114 106 114 F T F <---
66612024-02-06T22:18:25.308Z 19 380-399 60 58 60 115 113 115 F F F <---
66622024-02-06T22:18:25.308Z 20 400-419 60 58 60 115 113 115 F F F <---
66632024-02-06T22:18:25.308Z 21 420-439 57 57 57 112 112 112 F F F
66642024-02-06T22:18:25.308Z 22 440-459 58 58 58 113 113 113 F F F
66652024-02-06T22:18:25.308Z 23 460-479 58 58 58 113 113 113 F F F
66662024-02-06T22:18:25.308Z 24 480-499 58 58 58 113 113 113 F F F
66672024-02-06T22:18:25.308Z 25 500-519 58 58 58 113 113 113 F F F
66682024-02-06T22:18:25.308Z 26 520-539 60 58 60 114 113 114 T T T <---
66692024-02-06T22:18:25.308Z 27 540-559 48 48 48 93 93 93 F F F
66702024-02-06T22:18:25.308Z 28 560-579 58 58 58 113 113 113 F F F
66712024-02-06T22:18:25.308Z 29 580-599 58 58 58 113 113 113 F F F
66722024-02-06T22:18:25.308ZMax gen: 60, Max flush: 115
66732024-02-06T22:18:25.308ZError: Difference in extent metadata found!
66742024-02-06T22:18:25.308ZA Difference in extent metadata is expected here
66752024-02-06T22:18:25.308ZOn loop 30
66762024-02-06T22:18:25.309Z
66772024-02-06T22:18:25.312ZVerifying data now
66782024-02-06T22:18:31.053ZNo change after verify
66792024-02-06T22:18:31.061ZLoop: 30 Downstairs dump after verify (and repair):
66802024-02-06T22:18:31.070ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66812024-02-06T22:18:31.070Z 0 000-019 61 61 61 116 116 116 F F F
66822024-02-06T22:18:31.071Z 1 020-039 61 61 61 116 116 116 F F F
66832024-02-06T22:18:31.071Z 2 040-059 61 61 61 116 116 116 F F F
66842024-02-06T22:18:31.071Z 3 060-079 58 58 58 113 113 113 F F F
66852024-02-06T22:18:31.071Z 4 080-099 61 61 61 116 116 116 F F F
66862024-02-06T22:18:31.071Z 5 100-119 57 57 57 112 112 112 F F F
66872024-02-06T22:18:31.076Z 6 120-139 50 50 50 98 98 98 F F F
66882024-02-06T22:18:31.076Z 7 140-159 60 60 60 114 114 114 F F F
66892024-02-06T22:18:31.076Z 8 160-179 61 61 61 116 116 116 F F F
66902024-02-06T22:18:31.076Z 9 180-199 61 61 61 116 116 116 F F F
66912024-02-06T22:18:31.076Z 10 200-219 61 61 61 116 116 116 F F F
66922024-02-06T22:18:31.076Z 11 220-239 61 61 61 116 116 116 F F F
66932024-02-06T22:18:31.077Z 12 240-259 60 60 60 114 114 114 F F F
66942024-02-06T22:18:31.077Z 13 260-279 58 58 58 113 113 113 F F F
66952024-02-06T22:18:31.077Z 14 280-299 57 57 57 112 112 112 F F F
66962024-02-06T22:18:31.077Z 15 300-319 58 58 58 113 113 113 F F F
66972024-02-06T22:18:31.077Z 16 320-339 58 58 58 113 113 113 F F F
66982024-02-06T22:18:31.077Z 17 340-359 61 61 61 116 116 116 F F F
66992024-02-06T22:18:31.077Z 18 360-379 60 60 60 114 114 114 F F F
67002024-02-06T22:18:31.077Z 19 380-399 60 60 60 115 115 115 F F F
67012024-02-06T22:18:31.077Z 20 400-419 60 60 60 115 115 115 F F F
67022024-02-06T22:18:31.077Z 21 420-439 57 57 57 112 112 112 F F F
67032024-02-06T22:18:31.077Z 22 440-459 58 58 58 113 113 113 F F F
67042024-02-06T22:18:31.077Z 23 460-479 58 58 58 113 113 113 F F F
67052024-02-06T22:18:31.077Z 24 480-499 58 58 58 113 113 113 F F F
67062024-02-06T22:18:31.077Z 25 500-519 58 58 58 113 113 113 F F F
67072024-02-06T22:18:31.077Z 26 520-539 61 61 61 116 116 116 F F F
67082024-02-06T22:18:31.077Z 27 540-559 48 48 48 93 93 93 F F F
67092024-02-06T22:18:31.077Z 28 560-579 58 58 58 113 113 113 F F F
67102024-02-06T22:18:31.077Z 29 580-599 58 58 58 113 113 113 F F F
67112024-02-06T22:18:31.077ZMax gen: 61, Max flush: 116
67122024-02-06T22:18:31.077Z
67132024-02-06T22:18:31.077ZBegin loop 31
67142024-02-06T22:18:31.077ZDownstairs to restart: 2
67152024-02-06T22:18:31.092Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.311657168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383}
67162024-02-06T22:18:31.101Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.311942409Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383}
67172024-02-06T22:18:31.101ZCrucible runtime is spawned
67182024-02-06T22:18:31.101Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.311993122Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383}
67192024-02-06T22:18:31.101Z{"msg":"Crucible 77945b03-328a-4ab0-9370-4204f44e34fa has session id: 5e7db480-2a39-434e-b904-33104360083f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312018759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67202024-02-06T22:18:31.101Z{"msg":"Upstairs opts: Upstairs UUID: 77945b03-328a-4ab0-9370-4204f44e34fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312065572Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67212024-02-06T22:18:31.101Z{"msg":"Crucible stats registered with UUID: 77945b03-328a-4ab0-9370-4204f44e34fa","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312102048Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67222024-02-06T22:18:31.101Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312135763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383}
67232024-02-06T22:18:31.101Z{"msg":"77945b03-328a-4ab0-9370-4204f44e34fa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.31216Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67242024-02-06T22:18:31.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312195475Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67252024-02-06T22:18:31.101Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312240189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67262024-02-06T22:18:31.101Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312268055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67272024-02-06T22:18:31.102Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312295032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67282024-02-06T22:18:31.102Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312336776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67292024-02-06T22:18:31.102Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312364302Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67302024-02-06T22:18:31.102Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312389159Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67312024-02-06T22:18:31.102Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312419655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67322024-02-06T22:18:31.102Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.31245652Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67332024-02-06T22:18:31.102Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312491855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67342024-02-06T22:18:31.102Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312518441Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67352024-02-06T22:18:31.102Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:30.312787625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67362024-02-06T22:18:31.102Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:30.312831499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67372024-02-06T22:18:31.102Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:30.312861955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67382024-02-06T22:18:31.102Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312894851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67392024-02-06T22:18:31.102Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312930776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67402024-02-06T22:18:31.102Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.312952223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67412024-02-06T22:18:31.102Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.31319243Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67422024-02-06T22:18:31.102Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.313240544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67432024-02-06T22:18:31.102Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.313294637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67442024-02-06T22:18:31.102Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.31334568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67452024-02-06T22:18:31.102Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.313868839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67462024-02-06T22:18:31.102Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.313907164Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67472024-02-06T22:18:31.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.314504993Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67482024-02-06T22:18:31.102Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.314542588Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67492024-02-06T22:18:31.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.314558526Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67502024-02-06T22:18:31.103Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.314571604Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67512024-02-06T22:18:31.103Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.314593761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67522024-02-06T22:18:31.103Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:30.314608419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67532024-02-06T22:18:32.100Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.314374558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67542024-02-06T22:18:32.118Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.314589829Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67552024-02-06T22:18:32.118Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.315344087Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67562024-02-06T22:18:32.118Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.315381402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67572024-02-06T22:18:32.118Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.315816703Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67582024-02-06T22:18:32.119Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316371418Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67592024-02-06T22:18:32.119Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316410593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67602024-02-06T22:18:32.119Z{"msg":"[0]R flush_numbers[0..12]: [116, 116, 116, 113, 116, 112, 98, 114, 116, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316437049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67612024-02-06T22:18:32.119Z{"msg":"[0]R generation[0..12]: [61, 61, 61, 58, 61, 57, 50, 60, 61, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316472844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67622024-02-06T22:18:32.119Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316491172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67632024-02-06T22:18:32.119Z{"msg":"[1]R flush_numbers[0..12]: [116, 116, 116, 113, 116, 112, 98, 114, 116, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316516838Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67642024-02-06T22:18:32.119Z{"msg":"[1]R generation[0..12]: [61, 61, 61, 58, 61, 57, 50, 60, 61, 61, 61, 61]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
67652024-02-06T22:18:32.119Z,"time":"2024-02-06T22:18:31.316539405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67662024-02-06T22:18:32.119Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316565232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67672024-02-06T22:18:32.120Z{"msg":"[2]R flush_numbers[0..12]: [116, 116, 116, 113, 116, 112, 98, 114, 116, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316598837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67682024-02-06T22:18:32.120Z{"msg":"[2]R generation[0..12]: [61, 61, 61, 58, 61, 57, 50, 60, 61, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316620934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67692024-02-06T22:18:32.120Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.31664978Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67702024-02-06T22:18:32.120Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316668678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67712024-02-06T22:18:32.120Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316698744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67722024-02-06T22:18:32.120Z{"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316716661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67732024-02-06T22:18:32.120Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316746137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67742024-02-06T22:18:32.120Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316763625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67752024-02-06T22:18:32.120Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316791851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67762024-02-06T22:18:32.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316809919Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67772024-02-06T22:18:32.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316839155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67782024-02-06T22:18:32.121Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316858132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67792024-02-06T22:18:32.121Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316885888Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67802024-02-06T22:18:32.121Z{"msg":"77945b03-328a-4ab0-9370-4204f44e34fa is now active with session: 5e7db480-2a39-434e-b904-33104360083f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316901377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67812024-02-06T22:18:32.121Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316930762Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383,"session_id":"5e7db480-2a39-434e-b904-33104360083f"}
67822024-02-06T22:18:32.121Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67832024-02-06T22:18:32.121ZThe guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:31.316953869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383}
67842024-02-06T22:18:32.122ZLoading write count information from file "/tmp/test_repair_verify.data"
67852024-02-06T22:18:32.122ZRun Repair workload
67862024-02-06T22:18:32.122Z01/30 Write block 333 len 32768 data: 5 8 8 9 8 9 7 7
67872024-02-06T22:18:32.122Z02/30 Flush
67882024-02-06T22:18:32.122Z03/30 Write block 206 len 36864 data: 8 7 7 10 9 7 6 6 4
67892024-02-06T22:18:32.122Z04/30 Flush
67902024-02-06T22:18:32.122Z05/30 Write block 300 len 28672 data: 6 5 5 3 3 6 6
67912024-02-06T22:18:32.122Z06/30 Write block 191 len 24576 data: 4 4 4 5 5 7
67922024-02-06T22:18:32.123Z07/30 Read block 253 len 32768
67932024-02-06T22:18:32.138Z08/30 Read block 122 len 16384
67942024-02-06T22:18:32.138Z09/30 Write block 59 len 32768 data: 9 9 7 5 4 5 5 6
67952024-02-06T22:18:32.138Z10/30 Write block 114 len 16384 data: 9 8 7 8
67962024-02-06T22:18:32.138Z11/30 Read block 329 len 20480
67972024-02-06T22:18:32.138Z12/30 Write block 405 len 32768 data: 3 4 3 4 6 7 8 5
67982024-02-06T22:18:32.138Z13/30 Read block 427 len 36864
67992024-02-06T22:18:32.139Z14/30 Write block 529 len 16384 data: 7 8 8 7
68002024-02-06T22:18:32.139Z15/30 Write block 1 len 16384 data: 2 4 6 6
68012024-02-06T22:18:32.139Z16/30 Write block 554 len 4096 data: 4
68022024-02-06T22:18:32.139Z17/30 Write block 24 len 28672 data: 5 6 6 6 6 5 5
68032024-02-06T22:18:32.139Z18/30 Write block 274 len 8192 data: 3 5
68042024-02-06T22:18:32.139Z19/30 Read block 368 len 32768
68052024-02-06T22:18:32.139Z20/30 Read block 525 len 20480
68062024-02-06T22:18:32.140Z21/30 Read block 79 len 12288
68072024-02-06T22:18:32.140Z22/30 Write block 319 len 4096 data: 6
68082024-02-06T22:18:32.140Z23/30 Read block 382 len 16384
68092024-02-06T22:18:32.140Z24/30 Read block 317 len 4096
68102024-02-06T22:18:32.140Z25/30 Write block 233 len 20480 data: 4 5 5 5 5
68112024-02-06T22:18:32.140Z26/30 Read block 572 len 4096
68122024-02-06T22:18:32.140Z27/30 Read block 534 len 24576
68132024-02-06T22:18:32.140Z28/30 Write block 487 len 4096 data: 3
68142024-02-06T22:18:32.141Z29/30 Read block 568 len 16384
68152024-02-06T22:18:32.141Z30/30 Read block 15 len 16384
68162024-02-06T22:18:32.141Z----------------------------------------------------------------
68172024-02-06T22:18:32.141Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:26
68182024-02-06T22:18:32.141ZStates: Active Active Active
68192024-02-06T22:18:32.141ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
68202024-02-06T22:18:32.141Z 5 Acked 1004 Write 7 Done Done Sent false
68212024-02-06T22:18:32.141Z 6 Acked 1005 Write 6 Done Done Sent false
68222024-02-06T22:18:32.141Z 7 Acked 1006 Read 8 Done Done Sent false
68232024-02-06T22:18:32.142Z 8 Acked 1007 Read 4 Done Done Sent false
68242024-02-06T22:18:32.142Z 9 Acked 1008 Write 8 Done Done Sent false
68252024-02-06T22:18:32.142Z 10 Acked 1009 Write 4 Done Done Sent false
68262024-02-06T22:18:32.142Z 11 Acked 1010 Read 5 Done Done Sent false
68272024-02-06T22:18:32.142Z 12 Acked 1011 Write 8 Done Done Sent false
68282024-02-06T22:18:32.142Z 13 Acked 1012 Read 9 Done Done Sent false
68292024-02-06T22:18:32.142Z 14 Acked 1013 Write 4 Done Done Sent false
68302024-02-06T22:18:32.142Z 15 Acked 1014 Write 4 Done Done Sent false
68312024-02-06T22:18:32.143Z 16 Acked 1015 Write 1 Done Done Sent false
68322024-02-06T22:18:32.143Z 17 Acked 1016 Write 7 Done Done Sent false
68332024-02-06T22:18:32.143Z 18 Acked 1017 Write 2 Done Done Sent false
68342024-02-06T22:18:32.143Z 19 Acked 1018 Read 8 Done Done Sent false
68352024-02-06T22:18:32.143Z 20 Acked 1019 Read 5 Done Done Sent false
68362024-02-06T22:18:32.143Z 21 Acked 1020 Read 3 Done Done Sent false
68372024-02-06T22:18:32.143Z 22 Acked 1021 Write 1 Done Done Sent false
68382024-02-06T22:18:32.143Z 23 Acked 1022 Read 4 Done Done Sent false
68392024-02-06T22:18:32.144Z 24 Acked 1023 Read 1 Done Done Sent false
68402024-02-06T22:18:32.144Z 25 Acked 1024 Write 5 Done Done Sent false
68412024-02-06T22:18:32.144Z 26 Acked 1025 Read 1 Done Done Sent false
68422024-02-06T22:18:32.144Z 27 Acked 1026 Read 6 Done Done Sent false
68432024-02-06T22:18:32.144Z 28 Acked 1027 Write 1 Done Done Sent false
68442024-02-06T22:18:32.144Z 29 Acked 1028 Read 4 Done Done Sent false
68452024-02-06T22:18:32.144Z 30 Acked 1029 Read 4 Done Sent Sent false
68462024-02-06T22:18:32.144Z STATES DS:0 DS:1 DS:2 TOTAL
68472024-02-06T22:18:32.144Z New 0 0 0 0
68482024-02-06T22:18:32.145Z Sent 0 1 26 27
68492024-02-06T22:18:32.145Z Done 26 25 0 51
68502024-02-06T22:18:32.145Z Skipped 0 0 0 0
68512024-02-06T22:18:32.145Z Error 0 0 0 0
68522024-02-06T22:18:32.145ZLast Flush: 1003 1003 1003
68532024-02-06T22:18:32.145ZDownstairs last five completed: 1003 1002 1001 1000
68542024-02-06T22:18:32.145ZUpstairs last five completed: 30 29 28 27 26
68552024-02-06T22:18:32.145Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:31.345773656Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1383}
68562024-02-06T22:18:32.146ZWrote out file "/tmp/test_repair_verify.data"
68572024-02-06T22:18:32.164Z
68582024-02-06T22:18:32.164ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68592024-02-06T22:18:32.164ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68602024-02-06T22:18:32.164Z 0 000-019 61 61 61 116 116 116 T T F <---
68612024-02-06T22:18:32.164Z 1 020-039 61 61 61 116 116 116 T T F <---
68622024-02-06T22:18:32.165Z 2 040-059 61 61 61 116 116 116 T T F <---
68632024-02-06T22:18:32.165Z 3 060-079 58 58 58 113 113 113 T T F <---
68642024-02-06T22:18:32.165Z 4 080-099 61 61 61 116 116 116 F F F
68652024-02-06T22:18:32.165Z 5 100-119 57 57 57 112 112 112 T T F <---
68662024-02-06T22:18:32.165Z 6 120-139 50 50 50 98 98 98 F F F
68672024-02-06T22:18:32.165Z 7 140-159 60 60 60 114 114 114 F F F
68682024-02-06T22:18:32.165Z 8 160-179 61 61 61 116 116 116 F F F
68692024-02-06T22:18:32.165Z 9 180-199 61 61 61 116 116 116 T T F <---
68702024-02-06T22:18:32.165Z 10 200-219 62 62 62 118 118 118 F F F
68712024-02-06T22:18:32.166Z 11 220-239 61 61 61 116 116 116 T T F <---
68722024-02-06T22:18:32.166Z 12 240-259 60 60 60 114 114 114 F F F
68732024-02-06T22:18:32.166Z 13 260-279 58 58 58 113 113 113 T T F <---
68742024-02-06T22:18:32.166Z 14 280-299 57 57 57 112 112 112 F F F
68752024-02-06T22:18:32.166Z 15 300-319 58 58 58 113 113 113 T T F <---
68762024-02-06T22:18:32.166Z 16 320-339 62 62 62 117 117 117 F F F
68772024-02-06T22:18:32.166Z 17 340-359 62 62 62 117 117 117 F F F
68782024-02-06T22:18:32.166Z 18 360-379 60 60 60 114 114 114 F F F
68792024-02-06T22:18:32.167Z 19 380-399 60 60 60 115 115 115 F F F
68802024-02-06T22:18:32.167Z 20 400-419 60 60 60 115 115 115 T T F <---
68812024-02-06T22:18:32.167Z 21 420-439 57 57 57 112 112 112 F F F
68822024-02-06T22:18:32.167Z 22 440-459 58 58 58 113 113 113 F F F
68832024-02-06T22:18:32.167Z 23 460-479 58 58 58 113 113 113 F F F
68842024-02-06T22:18:32.167Z 24 480-499 58 58 58 113 113 113 T T F <---
68852024-02-06T22:18:32.168Z 25 500-519 58 58 58 113 113 113 F F F
68862024-02-06T22:18:32.168Z 26 520-539 61 61 61 116 116 116 T T F <---
68872024-02-06T22:18:32.168Z 27 540-559 48 48 48 93 93 93 T T F <---
68882024-02-06T22:18:32.168Z 28 560-579 58 58 58 113 113 113 F F F
68892024-02-06T22:18:32.168Z 29 580-599 58 58 58 113 113 113 F F F
68902024-02-06T22:18:32.168ZMax gen: 62, Max flush: 118
68912024-02-06T22:18:32.169ZError: Difference in extent metadata found!
68922024-02-06T22:18:32.179ZA Difference in extent metadata is expected here
68932024-02-06T22:18:32.179ZOn loop 31
68942024-02-06T22:18:32.179Z
68952024-02-06T22:18:32.179ZVerifying data now
68962024-02-06T22:18:36.856ZNo change after verify
68972024-02-06T22:18:36.856ZLoop: 31 Downstairs dump after verify (and repair):
68982024-02-06T22:18:36.872ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68992024-02-06T22:18:36.872Z 0 000-019 63 63 63 119 119 119 F F F
69002024-02-06T22:18:36.872Z 1 020-039 63 63 63 119 119 119 F F F
69012024-02-06T22:18:36.872Z 2 040-059 63 63 63 119 119 119 F F F
69022024-02-06T22:18:36.872Z 3 060-079 63 63 63 119 119 119 F F F
69032024-02-06T22:18:36.872Z 4 080-099 61 61 61 116 116 116 F F F
69042024-02-06T22:18:36.872Z 5 100-119 63 63 63 119 119 119 F F F
69052024-02-06T22:18:36.872Z 6 120-139 50 50 50 98 98 98 F F F
69062024-02-06T22:18:36.872Z 7 140-159 60 60 60 114 114 114 F F F
69072024-02-06T22:18:36.873Z 8 160-179 61 61 61 116 116 116 F F F
69082024-02-06T22:18:36.873Z 9 180-199 63 63 63 119 119 119 F F F
69092024-02-06T22:18:36.876Z 10 200-219 62 62 62 118 118 118 F F F
69102024-02-06T22:18:36.876Z 11 220-239 63 63 63 119 119 119 F F F
69112024-02-06T22:18:36.876Z 12 240-259 60 60 60 114 114 114 F F F
69122024-02-06T22:18:36.876Z 13 260-279 63 63 63 119 119 119 F F F
69132024-02-06T22:18:36.876Z 14 280-299 57 57 57 112 112 112 F F F
69142024-02-06T22:18:36.876Z 15 300-319 63 63 63 119 119 119 F F F
69152024-02-06T22:18:36.876Z 16 320-339 62 62 62 117 117 117 F F F
69162024-02-06T22:18:36.876Z 17 340-359 62 62 62 117 117 117 F F F
69172024-02-06T22:18:36.877Z 18 360-379 60 60 60 114 114 114 F F F
69182024-02-06T22:18:36.877Z 19 380-399 60 60 60 115 115 115 F F F
69192024-02-06T22:18:36.877Z 20 400-419 63 63 63 119 119 119 F F F
69202024-02-06T22:18:36.877Z 21 420-439 57 57 57 112 112 112 F F F
69212024-02-06T22:18:36.877Z 22 440-459 58 58 58 113 113 113 F F F
69222024-02-06T22:18:36.877Z 23 460-479 58 58 58 113 113 113 F F F
69232024-02-06T22:18:36.877Z 24 480-499 63 63 63 119 119 119 F F F
69242024-02-06T22:18:36.877Z 25 500-519 58 58 58 113 113 113 F F F
69252024-02-06T22:18:36.877Z 26 520-539 63 63 63 119 119 119 F F F
69262024-02-06T22:18:36.877Z 27 540-559 63 63 63 119 119 119 F F F
69272024-02-06T22:18:36.877Z 28 560-579 58 58 58 113 113 113 F F F
69282024-02-06T22:18:36.877Z 29 580-599 58 58 58 113 113 113 F F F
69292024-02-06T22:18:36.877ZMax gen: 63, Max flush: 119
69302024-02-06T22:18:36.878Z
69312024-02-06T22:18:36.878ZBegin loop 32
69322024-02-06T22:18:36.878ZDownstairs to restart: 2
69332024-02-06T22:18:36.896Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.112662131Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391}
69342024-02-06T22:18:36.897Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.112917267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391}
69352024-02-06T22:18:36.897Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.112944113Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391}
69362024-02-06T22:18:36.905Z{"msg":"Crucible 9192da5f-c149-4459-9006-0ff6d48f0d50 has session id: 9c28406a-b6f3-432e-9aa7-053fe8aa0dd1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.112961791Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69372024-02-06T22:18:36.905Z{"msg":"Upstairs opts: Upstairs UUID: 9192da5f-c149-4459-9006-0ff6d48f0d50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
69382024-02-06T22:18:36.905Z,"time":"2024-02-06T22:18:36.113021383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69392024-02-06T22:18:36.906Z{"msg":"Crucible stats registered with UUID: 9192da5f-c149-4459-9006-0ff6d48f0d50","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113048529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69402024-02-06T22:18:36.906Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113088804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391}
69412024-02-06T22:18:36.906Z{"msg":"9192da5f-c149-4459-9006-0ff6d48f0d50 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.11319228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69422024-02-06T22:18:36.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113224476Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69432024-02-06T22:18:36.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113239953Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69442024-02-06T22:18:36.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.11326159Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69452024-02-06T22:18:36.907Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113283217Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69462024-02-06T22:18:36.907Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113316193Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69472024-02-06T22:18:36.907Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113331031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69482024-02-06T22:18:36.907Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113344919Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69492024-02-06T22:18:36.907Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113372775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69502024-02-06T22:18:36.907Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113387683Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69512024-02-06T22:18:36.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113491939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69522024-02-06T22:18:36.907Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:36.113548022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69532024-02-06T22:18:36.907Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:36.113609033Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69542024-02-06T22:18:36.907Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:36.1136358Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69552024-02-06T22:18:36.907Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113653098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69562024-02-06T22:18:36.907Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113676644Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69572024-02-06T22:18:36.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113701591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69582024-02-06T22:18:36.907Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.113727757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69592024-02-06T22:18:36.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.114158009Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69602024-02-06T22:18:36.907Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.114195904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69612024-02-06T22:18:36.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.114211802Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69622024-02-06T22:18:36.907Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.11422958Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69632024-02-06T22:18:36.907Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.114851196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69642024-02-06T22:18:36.907Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.114921306Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69652024-02-06T22:18:36.907Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.115427798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69662024-02-06T22:18:36.908Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.115480511Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69672024-02-06T22:18:36.908Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.115514746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69682024-02-06T22:18:36.908Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.115549701Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69692024-02-06T22:18:36.908Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.115567679Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69702024-02-06T22:18:36.908Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:36.115589346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69712024-02-06T22:18:37.894Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.114182203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69722024-02-06T22:18:37.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.114364189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69732024-02-06T22:18:37.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.115510764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69742024-02-06T22:18:37.899Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.115554908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69752024-02-06T22:18:37.899Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116041862Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69762024-02-06T22:18:37.899Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116570891Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69772024-02-06T22:18:37.900Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116612945Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69782024-02-06T22:18:37.900Z{"msg":"[0]R flush_numbers[0..12]: [119, 119, 119, 119, 116, 119, 98, 114, 116, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116641591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69792024-02-06T22:18:37.900Z{"msg":"[0]R generation[0..12]: [63, 63, 63, 63, 61, 63, 50, 60, 61, 63, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116669987Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69802024-02-06T22:18:37.900Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116690215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69812024-02-06T22:18:37.900Z{"msg":"[1]R flush_numbers[0..12]: [119, 119, 119, 119, 116, 119, 98, 114, 116, 119, 118, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116716621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69822024-02-06T22:18:37.900Z{"msg":"[1]R generation[0..12]: [63, 63, 63, 63, 61, 63, 50, 60, 61, 63, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116735268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69832024-02-06T22:18:37.900Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116762095Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69842024-02-06T22:18:37.900Z{"msg":"[2]R flush_numbers[0..12]: [119, 119, 119, 119, 116, 119, 98, 114, 116, 119, 118, 119]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
69852024-02-06T22:18:37.900Z,"time":"2024-02-06T22:18:37.116780852Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69862024-02-06T22:18:37.900Z{"msg":"[2]R generation[0..12]: [63, 63, 63, 63, 61, 63, 50, 60, 61, 63, 62, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116809538Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69872024-02-06T22:18:37.900Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116853812Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69882024-02-06T22:18:37.900Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.11687266Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69892024-02-06T22:18:37.900Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116890447Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69902024-02-06T22:18:37.900Z{"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116930702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69912024-02-06T22:18:37.900Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116955508Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69922024-02-06T22:18:37.900Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.116993854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69932024-02-06T22:18:37.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117011971Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69942024-02-06T22:18:37.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117028879Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69952024-02-06T22:18:37.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117055445Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69962024-02-06T22:18:37.900Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117074032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69972024-02-06T22:18:37.900Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117098299Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69982024-02-06T22:18:37.900Z{"msg":"9192da5f-c149-4459-9006-0ff6d48f0d50 is now active with session: 9c28406a-b6f3-432e-9aa7-053fe8aa0dd1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117115307Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
69992024-02-06T22:18:37.900Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117138674Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391,"session_id":"9c28406a-b6f3-432e-9aa7-053fe8aa0dd1"}
70002024-02-06T22:18:37.900Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:37.117153662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391}
70012024-02-06T22:18:37.901ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
70022024-02-06T22:18:37.901ZLoading write count information from file "/tmp/test_repair_verify.data"
70032024-02-06T22:18:37.901ZRun Repair workload
70042024-02-06T22:18:37.904Z01/30 Write block 130 len 24576 data: 6 5 4 5 5 5
70052024-02-06T22:18:37.904Z02/30 Write block 220 len 8192 data: 11 11
70062024-02-06T22:18:37.904Z03/30 Write block 56 len 4096 data: 13
70072024-02-06T22:18:37.904Z04/30 Write block 124 len 20480 data: 6 7 7 8 8
70082024-02-06T22:18:37.904Z05/30 Read block 234 len 36864
70092024-02-06T22:18:37.904Z06/30 Read block 107 len 40960
70102024-02-06T22:18:37.904Z07/30 Write block 244 len 28672 data: 4 4 4 8 9 10 9
70112024-02-06T22:18:37.907Z08/30 Write block 266 len 28672 data: 7 6 7 7 6 4 4
70122024-02-06T22:18:37.907Z09/30 Write block 4 len 12288 data: 7 6 5
70132024-02-06T22:18:37.907Z10/30 Read block 292 len 32768
70142024-02-06T22:18:37.907Z11/30 Read block 437 len 24576
70152024-02-06T22:18:37.911Z12/30 Write block 422 len 12288 data: 6 7 7
70162024-02-06T22:18:37.911Z13/30 Read block 427 len 12288
70172024-02-06T22:18:37.911Z14/30 Read block 555 len 28672
70182024-02-06T22:18:37.911Z15/30 Read block 102 len 40960
70192024-02-06T22:18:37.920Z16/30 Read block 460 len 28672
70202024-02-06T22:18:37.920Z17/30 Write block 562 len 28672 data: 4 4 6 7 6 8 8
70212024-02-06T22:18:37.920Z18/30 Write block 362 len 4096 data: 11
70222024-02-06T22:18:37.920Z19/30 Read block 208 len 28672
70232024-02-06T22:18:37.920Z20/30 Write block 169 len 8192 data: 6 5
70242024-02-06T22:18:37.921Z21/30 Write block 59 len 20480 data: 10 10 8 6 5
70252024-02-06T22:18:37.921Z22/30 Write block 267 len 4096 data: 7
70262024-02-06T22:18:37.921Z23/30 Read block 321 len 24576
70272024-02-06T22:18:37.930Z24/30 Write block 234 len 36864 data: 6 6 6 6 4 4 4 4 5
70282024-02-06T22:18:37.930Z25/30 Read block 589 len 28672
70292024-02-06T22:18:37.930Z26/30 Read block 48 len 40960
70302024-02-06T22:18:37.930Z27/30 Write block 176 len 24576 data: 5 7 7 10 10 10
70312024-02-06T22:18:37.930Z28/30 Write block 136 len 24576 data: 5 6 6 6 4 3
70322024-02-06T22:18:37.930Z29/30 Read block 357 len 32768
70332024-02-06T22:18:37.930Z30/30 Read block 139 len 16384
70342024-02-06T22:18:37.944Z----------------------------------------------------------------
70352024-02-06T22:18:37.944Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
70362024-02-06T22:18:37.944ZStates: Active Active Active
70372024-02-06T22:18:37.944ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
70382024-02-06T22:18:37.944Z 1 Acked 1000 Write 6 Done Done Done false
70392024-02-06T22:18:37.945Z 2 Acked 1001 Write 2 Done Done Done false
70402024-02-06T22:18:37.945Z 3 Acked 1002 Write 1 Done Done Done false
70412024-02-06T22:18:37.945Z 4 Acked 1003 Write 5 Done Done Done false
70422024-02-06T22:18:37.945Z 5 Acked 1004 Read 9 Done Done Sent false
70432024-02-06T22:18:37.945Z 6 Acked 1005 Read 10 Done Done Sent false
70442024-02-06T22:18:37.945Z 7 Acked 1006 Write 7 Done Done Sent false
70452024-02-06T22:18:37.945Z 8 Acked 1007 Write 7 Done Done Sent false
70462024-02-06T22:18:37.945Z 9 Acked 1008 Write 3 Done Done Sent false
70472024-02-06T22:18:37.945Z 10 Acked 1009 Read 8 Done Done Sent false
70482024-02-06T22:18:37.945Z 11 Acked 1010 Read 6 Done Done Sent false
70492024-02-06T22:18:37.945Z 12 Acked 1011 Write 3 Done Done Sent false
70502024-02-06T22:18:37.945Z 13 Acked 1012 Read 3 Done Done Sent false
70512024-02-06T22:18:37.945Z 14 Acked 1013 Read 7 Done Done Sent false
70522024-02-06T22:18:37.945Z 15 Acked 1014 Read 10 Done Done Sent false
70532024-02-06T22:18:37.946Z 16 Acked 1015 Read 7 Done Done Sent false
70542024-02-06T22:18:37.946Z 17 Acked 1016 Write 7 Done Done Sent false
70552024-02-06T22:18:37.946Z 18 Acked 1017 Write 1 Done Done Sent false
70562024-02-06T22:18:37.946Z 19 Acked 1018 Read 7 Done Done Sent false
70572024-02-06T22:18:37.946Z 20 Acked 1019 Write 2 Done Done Sent false
70582024-02-06T22:18:37.946Z 21 Acked 1020 Write 5 Done Done Sent false
70592024-02-06T22:18:37.946Z 22 Acked 1021 Write 1 Done Done Sent false
70602024-02-06T22:18:37.946Z 23 Acked 1022 Read 6 Done Done Sent false
70612024-02-06T22:18:37.946Z 24 Acked 1023 Write 9 Done Done Sent false
70622024-02-06T22:18:37.946Z 25 Acked 1024 Read 7 Done Done Sent false
70632024-02-06T22:18:37.946Z 26 Acked 1025 Read 10 Done Done Sent false
70642024-02-06T22:18:37.946Z 27 Acked 1026 Write 6 Done Done Sent false
70652024-02-06T22:18:37.946Z 28 Acked 1027 Write 6 Done Done Sent false
70662024-02-06T22:18:37.946Z 29 Acked 1028 Read 8 Done Done Sent false
70672024-02-06T22:18:37.946Z 30 Acked 1029 Read 4 Done Done Sent false
70682024-02-06T22:18:37.946Z STATES DS:0 DS:1 DS:2 TOTAL
70692024-02-06T22:18:37.946Z New 0 0 0 0
70702024-02-06T22:18:37.947Z Sent 0 0 26 26
70712024-02-06T22:18:37.947Z Done 30 30 4 64
70722024-02-06T22:18:37.947Z Skipped 0 0 0 0
70732024-02-06T22:18:37.947Z Error 0 0 0 0
70742024-02-06T22:18:37.947ZLast Flush: 0 0 0
70752024-02-06T22:18:37.947ZDownstairs last five completed:
70762024-02-06T22:18:37.947ZUpstairs last five completed: 30 29 28 27 26
70772024-02-06T22:18:37.947Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:37.146083283Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1391}
70782024-02-06T22:18:37.947ZWrote out file "/tmp/test_repair_verify.data"
70792024-02-06T22:18:37.947Z
70802024-02-06T22:18:37.948ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70812024-02-06T22:18:37.956ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70822024-02-06T22:18:37.956Z 0 000-019 63 63 63 119 119 119 T T F <---
70832024-02-06T22:18:37.956Z 1 020-039 63 63 63 119 119 119 F F F
70842024-02-06T22:18:37.956Z 2 040-059 63 63 63 119 119 119 T T T
70852024-02-06T22:18:37.956Z 3 060-079 63 63 63 119 119 119 T T F <---
70862024-02-06T22:18:37.962Z 4 080-099 61 61 61 116 116 116 F F F
70872024-02-06T22:18:37.962Z 5 100-119 63 63 63 119 119 119 F F F
70882024-02-06T22:18:37.962Z 6 120-139 50 50 50 98 98 98 T T T
70892024-02-06T22:18:37.962Z 7 140-159 60 60 60 114 114 114 T T F <---
70902024-02-06T22:18:37.962Z 8 160-179 61 61 61 116 116 116 T T F <---
70912024-02-06T22:18:37.962Z 9 180-199 63 63 63 119 119 119 T T F <---
70922024-02-06T22:18:37.963Z 10 200-219 62 62 62 118 118 118 F F F
70932024-02-06T22:18:37.963Z 11 220-239 63 63 63 119 119 119 T T T
70942024-02-06T22:18:37.963Z 12 240-259 60 60 60 114 114 114 T T F <---
70952024-02-06T22:18:37.963Z 13 260-279 63 63 63 119 119 119 T T F <---
70962024-02-06T22:18:37.963Z 14 280-299 57 57 57 112 112 112 F F F
70972024-02-06T22:18:37.963Z 15 300-319 63 63 63 119 119 119 F F F
70982024-02-06T22:18:37.963Z 16 320-339 62 62 62 117 117 117 F F F
70992024-02-06T22:18:37.964Z 17 340-359 62 62 62 117 117 117 F F F
71002024-02-06T22:18:37.964Z 18 360-379 60 60 60 114 114 114 T T F <---
71012024-02-06T22:18:37.964Z 19 380-399 60 60 60 115 115 115 F F F
71022024-02-06T22:18:37.964Z 20 400-419 63 63 63 119 119 119 F F F
71032024-02-06T22:18:37.964Z 21 420-439 57 57 57 112 112 112 T T F <---
71042024-02-06T22:18:37.964Z 22 440-459 58 58 58 113 113 113 F F F
71052024-02-06T22:18:37.964Z 23 460-479 58 58 58 113 113 113 F F F
71062024-02-06T22:18:37.965Z 24 480-499 63 63 63 119 119 119 F F F
71072024-02-06T22:18:37.965Z 25 500-519 58 58 58 113 113 113 F F F
71082024-02-06T22:18:37.965Z 26 520-539 63 63 63 119 119 119 F F F
71092024-02-06T22:18:37.965Z 27 540-559 63 63 63 119 119 119 F F F
71102024-02-06T22:18:37.965Z 28 560-579 58 58 58 113 113 113 T T F <---
71112024-02-06T22:18:37.965Z 29 580-599 58 58 58 113 113 113 F F F
71122024-02-06T22:18:37.965ZMax gen: 63, Max flush: 119
71132024-02-06T22:18:37.965ZError: Difference in extent metadata found!
71142024-02-06T22:18:37.966ZA Difference in extent metadata is expected here
71152024-02-06T22:18:37.966ZOn loop 32
71162024-02-06T22:18:37.966Z
71172024-02-06T22:18:38.380ZVerifying data now
71182024-02-06T22:18:43.622ZNo change after verify
71192024-02-06T22:18:43.623ZLoop: 32 Downstairs dump after verify (and repair):
71202024-02-06T22:18:43.638ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
71212024-02-06T22:18:43.638Z 0 000-019 64 64 64 120 120 120 F F F
71222024-02-06T22:18:43.638Z 1 020-039 63 63 63 119 119 119 F F F
71232024-02-06T22:18:43.638Z 2 040-059 64 64 64 120 120 120 F F F
71242024-02-06T22:18:43.638Z 3 060-079 64 64 64 120 120 120 F F F
71252024-02-06T22:18:43.638Z 4 080-099 61 61 61 116 116 116 F F F
71262024-02-06T22:18:43.641Z 5 100-119 63 63 63 119 119 119 F F F
71272024-02-06T22:18:43.641Z 6 120-139 64 64 64 120 120 120 F F F
71282024-02-06T22:18:43.641Z 7 140-159 64 64 64 120 120 120 F F F
71292024-02-06T22:18:43.641Z 8 160-179 64 64 64 120 120 120 F F F
71302024-02-06T22:18:43.641Z 9 180-199 64 64 64 120 120 120 F F F
71312024-02-06T22:18:43.641Z 10 200-219 62 62 62 118 118 118 F F F
71322024-02-06T22:18:43.641Z 11 220-239 64 64 64 120 120 120 F F F
71332024-02-06T22:18:43.642Z 12 240-259 64 64 64 120 120 120 F F F
71342024-02-06T22:18:43.642Z 13 260-279 64 64 64 120 120 120 F F F
71352024-02-06T22:18:43.642Z 14 280-299 57 57 57 112 112 112 F F F
71362024-02-06T22:18:43.642Z 15 300-319 63 63 63 119 119 119 F F F
71372024-02-06T22:18:43.642Z 16 320-339 62 62 62 117 117 117 F F F
71382024-02-06T22:18:43.642Z 17 340-359 62 62 62 117 117 117 F F F
71392024-02-06T22:18:43.642Z 18 360-379 64 64 64 120 120 120 F F F
71402024-02-06T22:18:43.642Z 19 380-399 60 60 60 115 115 115 F F F
71412024-02-06T22:18:43.642Z 20 400-419 63 63 63 119 119 119 F F F
71422024-02-06T22:18:43.642Z 21 420-439 64 64 64 120 120 120 F F F
71432024-02-06T22:18:43.642Z 22 440-459 58 58 58 113 113 113 F F F
71442024-02-06T22:18:43.642Z 23 460-479 58 58 58 113 113 113 F F F
71452024-02-06T22:18:43.642Z 24 480-499 63 63 63 119 119 119 F F F
71462024-02-06T22:18:43.642Z 25 500-519 58 58 58 113 113 113 F F F
71472024-02-06T22:18:43.642Z 26 520-539 63 63 63 119 119 119 F F F
71482024-02-06T22:18:43.642Z 27 540-559 63 63 63 119 119 119 F F F
71492024-02-06T22:18:43.642Z 28 560-579 64 64 64 120 120 120 F F F
71502024-02-06T22:18:43.642Z 29 580-599 58 58 58 113 113 113 F F F
71512024-02-06T22:18:43.642ZMax gen: 64, Max flush: 120
71522024-02-06T22:18:43.645Z
71532024-02-06T22:18:43.645ZBegin loop 33
71542024-02-06T22:18:43.645ZDownstairs to restart: 2
71552024-02-06T22:18:43.662Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
71562024-02-06T22:18:43.666Z,"time":"2024-02-06T22:18:42.880741801Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399}
71572024-02-06T22:18:43.667Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881167563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399}
71582024-02-06T22:18:43.667Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881218616Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399}
71592024-02-06T22:18:43.667Z{"msg":"Crucible be0e873c-0b87-4db6-80ab-7ff7993457d6 has session id: 6249e8e6-f281-48e6-bf1e-2258966b8543","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.88126288Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71602024-02-06T22:18:43.667Z{"msg":"Upstairs opts: Upstairs UUID: be0e873c-0b87-4db6-80ab-7ff7993457d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881323152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71612024-02-06T22:18:43.667Z{"msg":"Crucible stats registered with UUID: be0e873c-0b87-4db6-80ab-7ff7993457d6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881368536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71622024-02-06T22:18:43.667Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881401072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399}
71632024-02-06T22:18:43.667Z{"msg":"be0e873c-0b87-4db6-80ab-7ff7993457d6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881442156Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71642024-02-06T22:18:43.667Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881467823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71652024-02-06T22:18:43.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881493969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71662024-02-06T22:18:43.667Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881520965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71672024-02-06T22:18:43.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881575648Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71682024-02-06T22:18:43.667Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881604894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71692024-02-06T22:18:43.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881630041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71702024-02-06T22:18:43.667Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881654367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71712024-02-06T22:18:43.667Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881702411Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71722024-02-06T22:18:43.667Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881746745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71732024-02-06T22:18:43.667Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.88178513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71742024-02-06T22:18:43.667Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881813686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71752024-02-06T22:18:43.667Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.881847401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71762024-02-06T22:18:43.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.882320607Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71772024-02-06T22:18:43.667Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.882364781Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71782024-02-06T22:18:43.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.882389438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71792024-02-06T22:18:43.668Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.882421794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71802024-02-06T22:18:43.668Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.882944363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71812024-02-06T22:18:43.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883096303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71822024-02-06T22:18:43.668Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883126148Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71832024-02-06T22:18:43.668Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883142736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71842024-02-06T22:18:43.668Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883497728Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71852024-02-06T22:18:43.668Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883549321Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71862024-02-06T22:18:43.668Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883578907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71872024-02-06T22:18:43.668Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883735386Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71882024-02-06T22:18:43.668Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883771481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71892024-02-06T22:18:43.668Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883787629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71902024-02-06T22:18:43.668Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.883801507Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71912024-02-06T22:18:43.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884283452Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71922024-02-06T22:18:43.674Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884332545Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71932024-02-06T22:18:43.674Z{"msg":"[0]R flush_numbers[0..12]: [120, 119, 120, 120, 116, 119, 120, 120, 120, 120, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884360392Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71942024-02-06T22:18:43.674Z{"msg":"[0]R generation[0..12]: [64, 63, 64, 64, 61, 63, 64, 64, 64, 64, 62, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884403686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71952024-02-06T22:18:43.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884423333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71962024-02-06T22:18:43.674Z{"msg":"[1]R flush_numbers[0..12]: [120, 119, 120, 120, 116, 119, 120, 120, 120, 120, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.88444705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71972024-02-06T22:18:43.674Z{"msg":"[1]R generation[0..12]: [64, 63, 64, 64, 61, 63, 64, 64, 64, 64, 62, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884461758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71982024-02-06T22:18:43.674Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884476226Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
71992024-02-06T22:18:43.674ZWait for a query_work_queue command to finish before sending IO
72002024-02-06T22:18:43.674Z{"msg":"[2]R flush_numbers[0..12]: [120, 119, 120, 120, 116, 119, 120, 120, 120, 120, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884503402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72012024-02-06T22:18:43.674Z{"msg":"[2]R generation[0..12]: [64, 63, 64, 64, 61, 63, 64, 64, 64, 64, 62, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884518351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72022024-02-06T22:18:43.674Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884533498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72032024-02-06T22:18:43.674Z{"msg":"Max found gen is 65","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884547036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72042024-02-06T22:18:43.674Z{"msg":"Generation requested: 66 >= found:65","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884573953Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72052024-02-06T22:18:43.675Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884588131Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72062024-02-06T22:18:43.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884601679Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72072024-02-06T22:18:43.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884625146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72082024-02-06T22:18:43.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884649353Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72092024-02-06T22:18:43.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884685758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72102024-02-06T22:18:43.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884723863Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72112024-02-06T22:18:43.675Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884762837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72122024-02-06T22:18:43.675Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884801312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72132024-02-06T22:18:43.675Z{"msg":"be0e873c-0b87-4db6-80ab-7ff7993457d6 is now active with session: 6249e8e6-f281-48e6-bf1e-2258966b8543","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884839077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72142024-02-06T22:18:43.675Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884872512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399,"session_id":"6249e8e6-f281-48e6-bf1e-2258966b8543"}
72152024-02-06T22:18:43.675Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:42.884905898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399}
72162024-02-06T22:18:43.675ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
72172024-02-06T22:18:43.675ZLoading write count information from file "/tmp/test_repair_verify.data"
72182024-02-06T22:18:43.675ZRun Repair workload
72192024-02-06T22:18:43.675Z01/30 Write block 596 len 8192 data: 8 7
72202024-02-06T22:18:43.675Z02/30 Flush
72212024-02-06T22:18:43.675Z03/30 Write block 546 len 32768 data: 6 4 4 5 4 4 4 4
72222024-02-06T22:18:43.675Z04/30 Flush
72232024-02-06T22:18:43.675Z05/30 Write block 7 len 8192 data: 6 6
72242024-02-06T22:18:43.675Z06/30 Write block 6 len 8192 data: 6 7
72252024-02-06T22:18:43.685Z07/30 Read block 432 len 28672
72262024-02-06T22:18:43.685Z08/30 Read block 30 len 4096
72272024-02-06T22:18:43.685Z09/30 Read block 251 len 24576
72282024-02-06T22:18:43.685Z10/30 Flush
72292024-02-06T22:18:43.685Z11/30 Write block 159 len 24576 data: 6 5 5 5 6 5
72302024-02-06T22:18:43.685Z12/30 Read block 175 len 24576
72312024-02-06T22:18:43.685Z13/30 Read block 167 len 24576
72322024-02-06T22:18:43.685Z14/30 Read block 366 len 32768
72332024-02-06T22:18:43.685Z15/30 Read block 168 len 12288
72342024-02-06T22:18:43.685Z16/30 Flush
72352024-02-06T22:18:43.685Z17/30 Write block 424 len 12288 data: 8 7 8
72362024-02-06T22:18:43.685Z18/30 Read block 314 len 8192
72372024-02-06T22:18:43.685Z19/30 Read block 213 len 24576
72382024-02-06T22:18:43.689Z20/30 Read block 272 len 40960
72392024-02-06T22:18:43.689Z21/30 Write block 532 len 28672 data: 8 8 6 5 6 5 4
72402024-02-06T22:18:43.689Z22/30 Read block 78 len 16384
72412024-02-06T22:18:43.689Z23/30 Flush
72422024-02-06T22:18:43.695Z24/30 Write block 228 len 28672 data: 5 4 4 6 6 5 7
72432024-02-06T22:18:43.695Z25/30 Read block 374 len 24576
72442024-02-06T22:18:43.698Z26/30 Write block 345 len 40960 data: 6 8 9 10 9 12 14 13 11 10
72452024-02-06T22:18:43.698Z27/30 Write block 511 len 32768 data: 4 3 6 6 6 6 6 5
72462024-02-06T22:18:43.699Z28/30 Read block 402 len 36864
72472024-02-06T22:18:43.699Z29/30 Write block 359 len 8192 data: 10 9
72482024-02-06T22:18:43.699Z30/30 Write block 152 len 16384 data: 7 9 8 9
72492024-02-06T22:18:43.699Z----------------------------------------------------------------
72502024-02-06T22:18:43.699Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:28
72512024-02-06T22:18:43.699ZStates: Active Active Active
72522024-02-06T22:18:43.699ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72532024-02-06T22:18:43.699Z 3 Acked 1002 Write 8 Done Done Sent false
72542024-02-06T22:18:43.699Z 4 Acked 1003 Flush 0 Done Done Sent false
72552024-02-06T22:18:43.699Z 5 Acked 1004 Write 2 Done Done Sent false
72562024-02-06T22:18:43.699Z 6 Acked 1005 Write 2 Done Done Sent false
72572024-02-06T22:18:43.699Z 7 Acked 1006 Read 7 Done Done Sent false
72582024-02-06T22:18:43.699Z 8 Acked 1007 Read 1 Done Done Sent false
72592024-02-06T22:18:43.699Z 9 Acked 1008 Read 6 Done Done Sent false
72602024-02-06T22:18:43.699Z 10 Acked 1009 Flush 0 Done Done Sent false
72612024-02-06T22:18:43.699Z 11 Acked 1010 Write 6 Done Done Sent false
72622024-02-06T22:18:43.699Z 12 Acked 1011 Read 6 Done Done Sent false
72632024-02-06T22:18:43.699Z 13 Acked 1012 Read 6 Done Done Sent false
72642024-02-06T22:18:43.699Z 14 Acked 1013 Read 8 Done Done Sent false
72652024-02-06T22:18:43.699Z 15 Acked 1014 Read 3 Done Done Sent false
72662024-02-06T22:18:43.699Z 16 Acked 1015 Flush 0 Done Done Sent false
72672024-02-06T22:18:43.699Z 17 Acked 1016 Write 3 Done Done Sent false
72682024-02-06T22:18:43.699Z 18 Acked 1017 Read 2 Done Done Sent false
72692024-02-06T22:18:43.700Z 19 Acked 1018 Read 6 Done Done Sent false
72702024-02-06T22:18:43.700Z 20 Acked 1019 Read 10 Done Done Sent false
72712024-02-06T22:18:43.700Z 21 Acked 1020 Write 7 Done Done Sent false
72722024-02-06T22:18:43.700Z 22 Acked 1021 Read 4 Done Done Sent false
72732024-02-06T22:18:43.700Z 23 Acked 1022 Flush 0 Done Done Sent false
72742024-02-06T22:18:43.700Z 24 Acked 1023 Write 7 Done Done Sent false
72752024-02-06T22:18:43.700Z 25 Acked 1024 Read 6 Done Done Sent false
72762024-02-06T22:18:43.700Z 26 Acked 1025 Write 10 Done Done Sent false
72772024-02-06T22:18:43.700Z 27 Acked 1026 Write 8 Done Done Sent false
72782024-02-06T22:18:43.700Z 28 Acked 1027 Read 9 Done Done Sent false
72792024-02-06T22:18:43.700Z 29 Acked 1028 Write 2 Done Sent Sent false
72802024-02-06T22:18:43.700Z 30 Acked 1029 Write 4 Sent Sent Sent false
72812024-02-06T22:18:43.700Z STATES DS:0 DS:1 DS:2 TOTAL
72822024-02-06T22:18:43.700Z New 0 0 0 0
72832024-02-06T22:18:43.700Z Sent 1 2 28 31
72842024-02-06T22:18:43.700Z Done 27 26 0 53
72852024-02-06T22:18:43.700Z Skipped 0 0 0 0
72862024-02-06T22:18:43.700Z Error 0 0 0 0
72872024-02-06T22:18:43.700ZLast Flush: 1022 1022 1001
72882024-02-06T22:18:43.700ZDownstairs last five completed: 1001 1000
72892024-02-06T22:18:43.700ZUpstairs last five completed: 30 29 28 27 26
72902024-02-06T22:18:43.700Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:42.915852217Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1399}
72912024-02-06T22:18:43.700ZWrote out file "/tmp/test_repair_verify.data"
72922024-02-06T22:18:43.709Z
72932024-02-06T22:18:43.709ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72942024-02-06T22:18:43.721ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72952024-02-06T22:18:43.721Z 0 000-019 66 66 64 123 123 120 F F F <---
72962024-02-06T22:18:43.721Z 1 020-039 63 63 63 119 119 119 F F F
72972024-02-06T22:18:43.721Z 2 040-059 64 64 64 120 120 120 F F F
72982024-02-06T22:18:43.721Z 3 060-079 64 64 64 120 120 120 F F F
72992024-02-06T22:18:43.721Z 4 080-099 61 61 61 116 116 116 F F F
73002024-02-06T22:18:43.721Z 5 100-119 63 63 63 119 119 119 F F F
73012024-02-06T22:18:43.721Z 6 120-139 64 64 64 120 120 120 F F F
73022024-02-06T22:18:43.721Z 7 140-159 66 66 64 124 124 120 T T F <---
73032024-02-06T22:18:43.721Z 8 160-179 66 66 64 124 124 120 F F F <---
73042024-02-06T22:18:43.721Z 9 180-199 64 64 64 120 120 120 F F F
73052024-02-06T22:18:43.724Z 10 200-219 62 62 62 118 118 118 F F F
73062024-02-06T22:18:43.724Z 11 220-239 64 64 64 120 120 120 T T F <---
73072024-02-06T22:18:43.725Z 12 240-259 64 64 64 120 120 120 F F F
73082024-02-06T22:18:43.725Z 13 260-279 64 64 64 120 120 120 F F F
73092024-02-06T22:18:43.725Z 14 280-299 57 57 57 112 112 112 F F F
73102024-02-06T22:18:43.725Z 15 300-319 63 63 63 119 119 119 F F F
73112024-02-06T22:18:43.725Z 16 320-339 62 62 62 117 117 117 F F F
73122024-02-06T22:18:43.725Z 17 340-359 62 62 62 117 117 117 T T F <---
73132024-02-06T22:18:43.725Z 18 360-379 64 64 64 120 120 120 T T F <---
73142024-02-06T22:18:43.725Z 19 380-399 60 60 60 115 115 115 F F F
73152024-02-06T22:18:43.725Z 20 400-419 63 63 63 119 119 119 F F F
73162024-02-06T22:18:43.725Z 21 420-439 66 66 64 125 125 120 F F F <---
73172024-02-06T22:18:43.725Z 22 440-459 58 58 58 113 113 113 F F F
73182024-02-06T22:18:43.725Z 23 460-479 58 58 58 113 113 113 F F F
73192024-02-06T22:18:43.725Z 24 480-499 63 63 63 119 119 119 F F F
73202024-02-06T22:18:43.725Z 25 500-519 58 58 58 113 113 113 T T F <---
73212024-02-06T22:18:43.725Z 26 520-539 66 66 63 125 125 119 F F F <---
73222024-02-06T22:18:43.725Z 27 540-559 66 66 63 122 122 119 F F F <---
73232024-02-06T22:18:43.725Z 28 560-579 64 64 64 120 120 120 F F F
73242024-02-06T22:18:43.725Z 29 580-599 66 66 66 121 121 121 F F F
73252024-02-06T22:18:43.725ZMax gen: 66, Max flush: 125
73262024-02-06T22:18:43.725ZError: Difference in extent metadata found!
73272024-02-06T22:18:43.725ZA Difference in extent metadata is expected here
73282024-02-06T22:18:43.725ZOn loop 33
73292024-02-06T22:18:43.725Z
73302024-02-06T22:18:43.728ZVerifying data now
73312024-02-06T22:18:49.951ZNo change after verify
73322024-02-06T22:18:49.965ZLoop: 33 Downstairs dump after verify (and repair):
73332024-02-06T22:18:49.965ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73342024-02-06T22:18:49.965Z 0 000-019 66 66 66 123 123 123 F F F
73352024-02-06T22:18:49.965Z 1 020-039 63 63 63 119 119 119 F F F
73362024-02-06T22:18:49.965Z 2 040-059 64 64 64 120 120 120 F F F
73372024-02-06T22:18:49.965Z 3 060-079 64 64 64 120 120 120 F F F
73382024-02-06T22:18:49.965Z 4 080-099 61 61 61 116 116 116 F F F
73392024-02-06T22:18:49.965Z 5 100-119 63 63 63 119 119 119 F F F
73402024-02-06T22:18:49.965Z 6 120-139 64 64 64 120 120 120 F F F
73412024-02-06T22:18:49.965Z 7 140-159 67 67 67 126 126 126 F F F
73422024-02-06T22:18:49.965Z 8 160-179 66 66 66 124 124 124 F F F
73432024-02-06T22:18:49.965Z 9 180-199 64 64 64 120 120 120 F F F
73442024-02-06T22:18:49.965Z 10 200-219 62 62 62 118 118 118 F F F
73452024-02-06T22:18:49.965Z 11 220-239 67 67 67 126 126 126 F F F
73462024-02-06T22:18:49.965Z 12 240-259 64 64 64 120 120 120 F F F
73472024-02-06T22:18:49.965Z 13 260-279 64 64 64 120 120 120 F F F
73482024-02-06T22:18:49.965Z 14 280-299 57 57 57 112 112 112 F F F
73492024-02-06T22:18:49.965Z 15 300-319 63 63 63 119 119 119 F F F
73502024-02-06T22:18:49.965Z 16 320-339 62 62 62 117 117 117 F F F
73512024-02-06T22:18:49.965Z 17 340-359 67 67 67 126 126 126 F F F
73522024-02-06T22:18:49.965Z 18 360-379 67 67 67 126 126 126 F F F
73532024-02-06T22:18:49.966Z 19 380-399 60 60 60 115 115 115 F F F
73542024-02-06T22:18:49.966Z 20 400-419 63 63 63 119 119 119 F F F
73552024-02-06T22:18:49.966Z 21 420-439 66 66 66 125 125 125 F F F
73562024-02-06T22:18:49.966Z 22 440-459 58 58 58 113 113 113 F F F
73572024-02-06T22:18:49.966Z 23 460-479 58 58 58 113 113 113 F F F
73582024-02-06T22:18:49.966Z 24 480-499 63 63 63 119 119 119 F F F
73592024-02-06T22:18:49.966Z 25 500-519 67 67 67 126 126 126 F F F
73602024-02-06T22:18:49.966Z 26 520-539 66 66 66 125 125 125 F F F
73612024-02-06T22:18:49.966Z 27 540-559 66 66 66 122 122 122 F F F
73622024-02-06T22:18:49.966Z 28 560-579 64 64 64 120 120 120 F F F
73632024-02-06T22:18:49.966Z 29 580-599 66 66 66 121 121 121 F F F
73642024-02-06T22:18:49.966ZMax gen: 67, Max flush: 126
73652024-02-06T22:18:49.966Z
73662024-02-06T22:18:49.966ZBegin loop 34
73672024-02-06T22:18:49.966ZDownstairs to restart: 0
73682024-02-06T22:18:49.966Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501256644Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407}
73692024-02-06T22:18:49.966Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501480744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407}
73702024-02-06T22:18:49.966Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501502841Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407}
73712024-02-06T22:18:49.966Z{"msg":"Crucible 7cf1acc8-a815-4400-9e4d-4b7a3a0b4b1b has session id: ec05f908-c8d5-4d20-b61a-ddc838807e9a","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501537596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73722024-02-06T22:18:49.966ZCrucible runtime is spawned
73732024-02-06T22:18:49.966Z{"msg":"Upstairs opts: Upstairs UUID: 7cf1acc8-a815-4400-9e4d-4b7a3a0b4b1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501595128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73742024-02-06T22:18:49.966Z{"msg":"Crucible stats registered with UUID: 7cf1acc8-a815-4400-9e4d-4b7a3a0b4b1b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501622604Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73752024-02-06T22:18:49.966Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50165482Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407}
73762024-02-06T22:18:49.966Z{"msg":"7cf1acc8-a815-4400-9e4d-4b7a3a0b4b1b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501736069Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73772024-02-06T22:18:49.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501767275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73782024-02-06T22:18:49.967Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501792961Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73792024-02-06T22:18:49.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501834536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73802024-02-06T22:18:49.967Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501861662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73812024-02-06T22:18:49.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501884989Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73822024-02-06T22:18:49.967Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501920614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73832024-02-06T22:18:49.967Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501945911Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73842024-02-06T22:18:49.967Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.501987725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73852024-02-06T22:18:49.967Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.502018191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73862024-02-06T22:18:49.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.502131846Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73872024-02-06T22:18:49.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.502205836Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73882024-02-06T22:18:49.967Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50224982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73892024-02-06T22:18:49.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.502772259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73902024-02-06T22:18:49.967Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.502810404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73912024-02-06T22:18:49.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50284187Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73922024-02-06T22:18:49.967Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.502880535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73932024-02-06T22:18:49.967Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.503426981Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73942024-02-06T22:18:49.967Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50357978Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73952024-02-06T22:18:49.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504028449Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73962024-02-06T22:18:49.967Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504070684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73972024-02-06T22:18:49.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50409871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73982024-02-06T22:18:49.967Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504136905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
73992024-02-06T22:18:49.967Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504157522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74002024-02-06T22:18:49.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504178339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74012024-02-06T22:18:49.968Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504192267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74022024-02-06T22:18:49.968Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504206465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74032024-02-06T22:18:49.968Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.504611601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74042024-02-06T22:18:49.968Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505042232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74052024-02-06T22:18:49.968Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505088566Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74062024-02-06T22:18:49.968Z{"msg":"[0]R flush_numbers[0..12]: [123, 119, 120, 120, 116, 119, 120, 126, 124, 120, 118, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505125441Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74072024-02-06T22:18:49.968Z{"msg":"[0]R generation[0..12]: [66, 63, 64, 64, 61, 63, 64, 67, 66, 64, 62, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505162326Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74082024-02-06T22:18:49.968Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505197222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74092024-02-06T22:18:49.968Z{"msg":"[1]R flush_numbers[0..12]: [123, 119, 120, 120, 116, 119, 120, 126, 124, 120, 118, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505234466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74102024-02-06T22:18:49.968Z{"msg":"[1]R generation[0..12]: [66, 63, 64, 64, 61, 63, 64, 67, 66, 64, 62, 67]","v":0,"name":Wait for a query_work_queue command to finish before sending IO
74112024-02-06T22:18:49.968Z"crucible","level":30,"time":"2024-02-06T22:18:47.505275821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74122024-02-06T22:18:49.968Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505317295Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74132024-02-06T22:18:49.968Z{"msg":"[2]R flush_numbers[0..12]: [123, 119, 120, 120, 116, 119, 120, 126, 124, 120, 118, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50535804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74142024-02-06T22:18:49.968Z{"msg":"[2]R generation[0..12]: [66, 63, 64, 64, 61, 63, 64, 67, 66, 64, 62, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505386946Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74152024-02-06T22:18:49.968Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505424781Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74162024-02-06T22:18:49.968Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505461066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74172024-02-06T22:18:49.968Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50549949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74182024-02-06T22:18:49.968Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505536656Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74192024-02-06T22:18:49.968Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505573301Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
74202024-02-06T22:18:49.968Z1407,"":"downstairs","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74212024-02-06T22:18:49.968Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505615725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74222024-02-06T22:18:49.968Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505644001Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74232024-02-06T22:18:49.968Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505658699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74242024-02-06T22:18:49.968Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505680126Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74252024-02-06T22:18:49.969Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505694034Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74262024-02-06T22:18:49.969Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505714312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74272024-02-06T22:18:49.969Z{"msg":"7cf1acc8-a815-4400-9e4d-4b7a3a0b4b1b is now active with session: ec05f908-c8d5-4d20-b61a-ddc838807e9a","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.50572784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74282024-02-06T22:18:49.969Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505740698Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407,"session_id":"ec05f908-c8d5-4d20-b61a-ddc838807e9a"}
74292024-02-06T22:18:49.969Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:47.505760165Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407}
74302024-02-06T22:18:49.969ZLoading write count information from file "/tmp/test_repair_verify.data"
74312024-02-06T22:18:49.969ZRun Repair workload
74322024-02-06T22:18:49.987Z01/30 Write block 175 len 8192 data: 6 6
74332024-02-06T22:18:49.987Z02/30 Read block 591 len 4096
74342024-02-06T22:18:49.987Z03/30 Read block 327 len 32768
74352024-02-06T22:18:49.987Z04/30 Flush
74362024-02-06T22:18:49.987Z05/30 Write block 91 len 40960 data: 4 2 2 3 4 4 4 5 5 5
74372024-02-06T22:18:49.987Z06/30 Read block 484 len 12288
74382024-02-06T22:18:49.987Z07/30 Write block 34 len 40960 data: 3 2 3 4 4 5 8 7 7 7
74392024-02-06T22:18:49.987Z08/30 Read block 69 len 12288
74402024-02-06T22:18:49.987Z09/30 Read block 369 len 32768
74412024-02-06T22:18:49.987Z10/30 Read block 237 len 28672
74422024-02-06T22:18:49.987Z11/30 Write block 559 len 16384 data: 4 4 4 5
74432024-02-06T22:18:49.988Z12/30 Read block 177 len 32768
74442024-02-06T22:18:49.988Z13/30 Flush
74452024-02-06T22:18:49.988Z14/30 Write block 503 len 16384 data: 8 6 6 7
74462024-02-06T22:18:49.988Z15/30 Write block 49 len 12288 data: 8 9 11
74472024-02-06T22:18:49.988Z16/30 Read block 500 len 28672
74482024-02-06T22:18:49.988Z17/30 Read block 4 len 20480
74492024-02-06T22:18:49.988Z18/30 Read block 549 len 16384
74502024-02-06T22:18:49.988Z19/30 Read block 154 len 40960
74512024-02-06T22:18:49.988Z20/30 Write block 415 len 20480 data: 7 8 10 10 9
74522024-02-06T22:18:49.988Z21/30 Flush
74532024-02-06T22:18:49.988Z22/30 Write block 126 len 32768 data: 8 9 9 6 7 6 5 6
74542024-02-06T22:18:49.988Z23/30 Read block 434 len 36864
74552024-02-06T22:18:49.988Z24/30 Read block 233 len 32768
74562024-02-06T22:18:49.988Z25/30 Read block 372 len 16384
74572024-02-06T22:18:49.988Z26/30 Read block 104 len 16384
74582024-02-06T22:18:49.988Z27/30 Read block 340 len 32768
74592024-02-06T22:18:49.988Z28/30 Write block 276 len 28672 data: 6 5 6 8 8 8 7
74602024-02-06T22:18:49.988Z29/30 Read block 95 len 8192
74612024-02-06T22:18:49.988Z30/30 Read block 223 len 8192
74622024-02-06T22:18:49.988Z----------------------------------------------------------------
74632024-02-06T22:18:49.988Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74642024-02-06T22:18:49.988ZStates: Active Active Active
74652024-02-06T22:18:49.988ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74662024-02-06T22:18:49.989Z 1 Acked 1000 Write 2 Done Done Done false
74672024-02-06T22:18:49.989Z 2 Acked 1001 Read 1 Done Done Done false
74682024-02-06T22:18:49.989Z 3 Acked 1002 Read 8 Done Done Done false
74692024-02-06T22:18:49.989Z 4 Acked 1003 Flush 0 Sent Done Done false
74702024-02-06T22:18:49.989Z 5 Acked 1004 Write 10 Sent Done Done false
74712024-02-06T22:18:49.989Z 6 Acked 1005 Read 3 Sent Done Done false
74722024-02-06T22:18:49.989Z 7 Acked 1006 Write 10 Sent Done Done false
74732024-02-06T22:18:49.989Z 8 Acked 1007 Read 3 Sent Done Done false
74742024-02-06T22:18:49.989Z 9 Acked 1008 Read 8 Sent Done Done false
74752024-02-06T22:18:49.989Z 10 Acked 1009 Read 7 Sent Done Done false
74762024-02-06T22:18:49.989Z 11 Acked 1010 Write 4 Sent Done Done false
74772024-02-06T22:18:49.989Z 12 Acked 1011 Read 8 Sent Done Done false
74782024-02-06T22:18:49.989Z 13 Acked 1012 Flush 0 Sent Done Done false
74792024-02-06T22:18:49.989Z 14 Acked 1013 Write 4 Sent Done Done false
74802024-02-06T22:18:49.989Z 15 Acked 1014 Write 3 Sent Done Done false
74812024-02-06T22:18:49.989Z 16 Acked 1015 Read 7 Sent Done Done false
74822024-02-06T22:18:49.989Z 17 Acked 1016 Read 5 Sent Done Done false
74832024-02-06T22:18:49.989Z 18 Acked 1017 Read 4 Sent Done Done false
74842024-02-06T22:18:49.989Z 19 Acked 1018 Read 10 Sent Done Done false
74852024-02-06T22:18:49.989Z 20 Acked 1019 Write 5 Sent Done Done false
74862024-02-06T22:18:49.989Z 21 Acked 1020 Flush 0 Sent Done Done false
74872024-02-06T22:18:49.989Z 22 Acked 1021 Write 8 Sent Done Done false
74882024-02-06T22:18:49.989Z 23 Acked 1022 Read 9 Sent Done Done false
74892024-02-06T22:18:49.989Z 24 Acked 1023 Read 8 Sent Done Done false
74902024-02-06T22:18:49.990Z 25 Acked 1024 Read 4 Sent Done Done false
74912024-02-06T22:18:49.990Z 26 Acked 1025 Read 4 Sent Done Done false
74922024-02-06T22:18:49.990Z 27 Acked 1026 Read 8 Sent Done Done false
74932024-02-06T22:18:49.990Z 28 Acked 1027 Write 7 Sent Done Done false
74942024-02-06T22:18:49.990Z 29 Acked 1028 Read 2 Sent Done Done false
74952024-02-06T22:18:49.990Z 30 Acked 1029 Read 2 Sent Sent Done false
74962024-02-06T22:18:49.990Z STATES DS:0 DS:1 DS:2 TOTAL
74972024-02-06T22:18:49.990Z New 0 0 0 0
74982024-02-06T22:18:49.990Z Sent 27 1 0 28
74992024-02-06T22:18:49.990Z Done 3 29 30 62
75002024-02-06T22:18:49.990Z Skipped 0 0 0 0
75012024-02-06T22:18:49.990Z Error 0 0 0 0
75022024-02-06T22:18:49.990ZLast Flush: 0 1020 1020
75032024-02-06T22:18:49.990ZDownstairs last five completed:
75042024-02-06T22:18:49.990ZUpstairs last five completed: 30 29 28 27 26
75052024-02-06T22:18:49.990Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:47.538565713Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1407}
75062024-02-06T22:18:49.990ZWrote out file "/tmp/test_repair_verify.data"
75072024-02-06T22:18:49.990Z
75082024-02-06T22:18:49.990ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
75092024-02-06T22:18:49.990ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75102024-02-06T22:18:49.990Z 0 000-019 66 66 66 123 123 123 F F F
75112024-02-06T22:18:49.990Z 1 020-039 63 68 68 119 128 128 F F F <---
75122024-02-06T22:18:49.990Z 2 040-059 64 68 68 120 129 129 F F F <---
75132024-02-06T22:18:49.991Z 3 060-079 64 64 64 120 120 120 F F F
75142024-02-06T22:18:49.991Z 4 080-099 61 68 68 116 128 128 F F F <---
75152024-02-06T22:18:49.991Z 5 100-119 63 68 68 119 128 128 F F F <---
75162024-02-06T22:18:49.991Z 6 120-139 64 64 64 120 120 120 F T T <---
75172024-02-06T22:18:49.991Z 7 140-159 67 67 67 126 126 126 F F F
75182024-02-06T22:18:49.991Z 8 160-179 66 68 68 124 127 127 T F F <---
75192024-02-06T22:18:49.991Z 9 180-199 64 64 64 120 120 120 F F F
75202024-02-06T22:18:49.991Z 10 200-219 62 62 62 118 118 118 F F F
75212024-02-06T22:18:49.991Z 11 220-239 67 67 67 126 126 126 F F F
75222024-02-06T22:18:49.991Z 12 240-259 64 64 64 120 120 120 F F F
75232024-02-06T22:18:49.991Z 13 260-279 64 64 64 120 120 120 F T T <---
75242024-02-06T22:18:49.991Z 14 280-299 57 57 57 112 112 112 F T T <---
75252024-02-06T22:18:49.991Z 15 300-319 63 63 63 119 119 119 F F F
75262024-02-06T22:18:49.991Z 16 320-339 62 62 62 117 117 117 F F F
75272024-02-06T22:18:49.991Z 17 340-359 67 67 67 126 126 126 F F F
75282024-02-06T22:18:49.991Z 18 360-379 67 67 67 126 126 126 F F F
75292024-02-06T22:18:49.991Z 19 380-399 60 60 60 115 115 115 F F F
75302024-02-06T22:18:49.991Z 20 400-419 63 68 68 119 129 129 F F F <---
75312024-02-06T22:18:49.991Z 21 420-439 66 66 66 125 125 125 F F F
75322024-02-06T22:18:50.003Z 22 440-459 58 58 58 113 113 113 F F F
75332024-02-06T22:18:50.003Z 23 460-479 58 58 58 113 113 113 F F F
75342024-02-06T22:18:50.003Z 24 480-499 63 63 63 119 119 119 F F F
75352024-02-06T22:18:50.003Z 25 500-519 67 68 68 126 129 129 F F F <---
75362024-02-06T22:18:50.003Z 26 520-539 66 66 66 125 125 125 F F F
75372024-02-06T22:18:50.003Z 27 540-559 66 68 68 122 128 128 F F F <---
75382024-02-06T22:18:50.003Z 28 560-579 64 68 68 120 128 128 F F F <---
75392024-02-06T22:18:50.003Z 29 580-599 66 66 66 121 121 121 F F F
75402024-02-06T22:18:50.003ZMax gen: 68, Max flush: 129
75412024-02-06T22:18:50.003ZError: Difference in extent metadata found!
75422024-02-06T22:18:50.003ZA Difference in extent metadata is expected here
75432024-02-06T22:18:50.003ZOn loop 34
75442024-02-06T22:18:50.003Z
75452024-02-06T22:18:50.003ZVerifying data now
75462024-02-06T22:18:53.953ZNo change after verify
75472024-02-06T22:18:53.953ZLoop: 34 Downstairs dump after verify (and repair):
75482024-02-06T22:18:53.987ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75492024-02-06T22:18:53.987Z 0 000-019 66 66 66 123 123 123 F F F
75502024-02-06T22:18:53.987Z 1 020-039 68 68 68 128 128 128 F F F
75512024-02-06T22:18:53.987Z 2 040-059 68 68 68 129 129 129 F F F
75522024-02-06T22:18:53.987Z 3 060-079 64 64 64 120 120 120 F F F
75532024-02-06T22:18:53.987Z 4 080-099 68 68 68 128 128 128 F F F
75542024-02-06T22:18:53.987Z 5 100-119 68 68 68 128 128 128 F F F
75552024-02-06T22:18:53.988Z 6 120-139 69 69 69 130 130 130 F F F
75562024-02-06T22:18:53.988Z 7 140-159 67 67 67 126 126 126 F F F
75572024-02-06T22:18:53.988Z 8 160-179 68 68 68 127 127 127 F F F
75582024-02-06T22:18:53.988Z 9 180-199 64 64 64 120 120 120 F F F
75592024-02-06T22:18:53.988Z 10 200-219 62 62 62 118 118 118 F F F
75602024-02-06T22:18:53.988Z 11 220-239 67 67 67 126 126 126 F F F
75612024-02-06T22:18:53.988Z 12 240-259 64 64 64 120 120 120 F F F
75622024-02-06T22:18:53.988Z 13 260-279 69 69 69 130 130 130 F F F
75632024-02-06T22:18:53.988Z 14 280-299 69 69 69 130 130 130 F F F
75642024-02-06T22:18:53.988Z 15 300-319 63 63 63 119 119 119 F F F
75652024-02-06T22:18:53.988Z 16 320-339 62 62 62 117 117 117 F F F
75662024-02-06T22:18:53.988Z 17 340-359 67 67 67 126 126 126 F F F
75672024-02-06T22:18:53.989Z 18 360-379 67 67 67 126 126 126 F F F
75682024-02-06T22:18:53.989Z 19 380-399 60 60 60 115 115 115 F F F
75692024-02-06T22:18:53.989Z 20 400-419 68 68 68 129 129 129 F F F
75702024-02-06T22:18:53.989Z 21 420-439 66 66 66 125 125 125 F F F
75712024-02-06T22:18:53.989Z 22 440-459 58 58 58 113 113 113 F F F
75722024-02-06T22:18:53.989Z 23 460-479 58 58 58 113 113 113 F F F
75732024-02-06T22:18:53.989Z 24 480-499 63 63 63 119 119 119 F F F
75742024-02-06T22:18:53.989Z 25 500-519 68 68 68 129 129 129 F F F
75752024-02-06T22:18:53.989Z 26 520-539 66 66 66 125 125 125 F F F
75762024-02-06T22:18:53.989Z 27 540-559 68 68 68 128 128 128 F F F
75772024-02-06T22:18:53.989Z 28 560-579 68 68 68 128 128 128 F F F
75782024-02-06T22:18:53.989Z 29 580-599 66 66 66 121 121 121 F F F
75792024-02-06T22:18:53.990ZMax gen: 69, Max flush: 130
75802024-02-06T22:18:53.990Z
75812024-02-06T22:18:53.990ZBegin loop 35
75822024-02-06T22:18:53.990ZDownstairs to restart: 2
75832024-02-06T22:18:54.015Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.205843215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415}
75842024-02-06T22:18:54.015Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"Crucible runtime is spawned
75852024-02-06T22:18:54.015Zx86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206186138Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415}
75862024-02-06T22:18:54.015Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206217464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415}
75872024-02-06T22:18:54.015Z{"msg":"Crucible 4bc62092-ea12-4f8f-a0a2-60d00596a96c has session id: 4848b222-71d1-42b6-a24b-50d8bf2ded74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206250759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75882024-02-06T22:18:54.015Z{"msg":"Upstairs opts: Upstairs UUID: 4bc62092-ea12-4f8f-a0a2-60d00596a96c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206285115Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75892024-02-06T22:18:54.015Z{"msg":"Crucible stats registered with UUID: 4bc62092-ea12-4f8f-a0a2-60d00596a96c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206301102Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75902024-02-06T22:18:54.015Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.20632108Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415}
75912024-02-06T22:18:54.015Z{"msg":"4bc62092-ea12-4f8f-a0a2-60d00596a96c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206336418Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75922024-02-06T22:18:54.015Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206359425Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75932024-02-06T22:18:54.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206398109Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75942024-02-06T22:18:54.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206434944Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75952024-02-06T22:18:54.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206481608Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75962024-02-06T22:18:54.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206518253Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75972024-02-06T22:18:54.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206553139Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75982024-02-06T22:18:54.016Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206580415Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
75992024-02-06T22:18:54.016Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206621989Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76002024-02-06T22:18:54.016Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206737524Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76012024-02-06T22:18:54.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206782317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76022024-02-06T22:18:54.016Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:53.206834721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76032024-02-06T22:18:54.016Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:53.206859417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76042024-02-06T22:18:54.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206888683Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76052024-02-06T22:18:54.017Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:53.206918179Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76062024-02-06T22:18:54.017Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206953714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76072024-02-06T22:18:54.017Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.206974432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76082024-02-06T22:18:54.017Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.207082197Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76092024-02-06T22:18:54.017Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.207459076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76102024-02-06T22:18:54.017Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.207500011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76112024-02-06T22:18:54.017Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.207541705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76122024-02-06T22:18:54.017Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.20757529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76132024-02-06T22:18:54.017Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208092931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76142024-02-06T22:18:54.017Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208224673Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76152024-02-06T22:18:54.017Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208630798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76162024-02-06T22:18:54.018Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208683111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76172024-02-06T22:18:54.018Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208715846Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76182024-02-06T22:18:54.018Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208753851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76192024-02-06T22:18:54.018Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208772539Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76202024-02-06T22:18:54.018Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:53.208795396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76212024-02-06T22:18:54.992Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.20845426Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76222024-02-06T22:18:54.996Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.208636225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76232024-02-06T22:18:54.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.20941297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76242024-02-06T22:18:54.997Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.209449835Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76252024-02-06T22:18:54.997Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.209886626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76262024-02-06T22:18:54.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210382759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76272024-02-06T22:18:54.997Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210418684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76282024-02-06T22:18:54.997Z{"msg":"[0]R flush_numbers[0..12]: [123, 128, 129, 120, 128, 128, 130, 126, 127, 120, 118, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210438411Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76292024-02-06T22:18:54.997Z{"msg":"[0]R generation[0..12]: [66, 68, 68, 64, 68, 68, 69, 67, 68, 64, 62, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210457059Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76302024-02-06T22:18:54.997Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210485485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76312024-02-06T22:18:54.997Z{"msg":"[1]R flush_numbers[0..12]: [123, 128, 129, 120, 128, 128, 130, 126, 127, 120, 118, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210505062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76322024-02-06T22:18:54.998Z{"msg":"[1]R generation[0..12]: [66, 68, 68, 64, 68, 68, 69, 67, 68, 64, 62, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210532689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76332024-02-06T22:18:54.998Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210566624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76342024-02-06T22:18:54.998Z{"msg":"[2]R flush_numbers[0..12]: [123, 128, 129, 120, 128, 128, 130, 126, 127, 120, 118, 126]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
76352024-02-06T22:18:54.998Ztime":"2024-02-06T22:18:54.210586062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76362024-02-06T22:18:54.998Z{"msg":"[2]R generation[0..12]: [66, 68, 68, 64, 68, 68, 69, 67, 68, 64, 62, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210615128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76372024-02-06T22:18:54.998Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210649593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76382024-02-06T22:18:54.998Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210668311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76392024-02-06T22:18:54.998Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210694597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76402024-02-06T22:18:54.998Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210712445Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76412024-02-06T22:18:54.998Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210737821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76422024-02-06T22:18:54.998Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210756039Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76432024-02-06T22:18:54.999Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210779705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76442024-02-06T22:18:54.999Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210799073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76452024-02-06T22:18:54.999Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210825209Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76462024-02-06T22:18:54.999Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210844267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76472024-02-06T22:18:54.999Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210868684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76482024-02-06T22:18:54.999Z{"msg":"4bc62092-ea12-4f8f-a0a2-60d00596a96c is now active with session: 4848b222-71d1-42b6-a24b-50d8bf2ded74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210887261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76492024-02-06T22:18:54.999Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210911568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415,"session_id":"4848b222-71d1-42b6-a24b-50d8bf2ded74"}
76502024-02-06T22:18:54.999Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:54.210929325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415}
76512024-02-06T22:18:54.999ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76522024-02-06T22:18:54.999ZLoading write count information from file "/tmp/test_repair_verify.data"
76532024-02-06T22:18:54.999ZRun Repair workload
76542024-02-06T22:18:54.999Z01/30 Write block 122 len 8192 data: 7 6
76552024-02-06T22:18:55.004Z02/30 Read block 404 len 20480
76562024-02-06T22:18:55.004Z03/30 Write block 235 len 16384 data: 7 7 7 5
76572024-02-06T22:18:55.004Z04/30 Write block 288 len 36864 data: 5 5 8 8 8 8 8 9 7
76582024-02-06T22:18:55.004Z05/30 Write block 507 len 32768 data: 7 6 5 5 5 4 7 7
76592024-02-06T22:18:55.004Z06/30 Read block 326 len 12288
76602024-02-06T22:18:55.004Z07/30 Write block 141 len 36864 data: 4 5 6 7 8 9 10 11 10
76612024-02-06T22:18:55.004Z08/30 Read block 158 len 12288
76622024-02-06T22:18:55.004Z09/30 Write block 417 len 16384 data: 11 11 10 8
76632024-02-06T22:18:55.004Z10/30 Read block 483 len 32768
76642024-02-06T22:18:55.008Z11/30 Write block 121 len 24576 data: 7 8 7 7 8 9
76652024-02-06T22:18:55.008Z12/30 Read block 26 len 32768
76662024-02-06T22:18:55.008Z13/30 Write block 565 len 36864 data: 8 7 9 9 6 6 9 10 8
76672024-02-06T22:18:55.008Z14/30 Read block 547 len 20480
76682024-02-06T22:18:55.012Z15/30 Write block 120 len 24576 data: 7 8 9 8 8 9
76692024-02-06T22:18:55.012Z16/30 Read block 355 len 16384
76702024-02-06T22:18:55.012Z17/30 Read block 319 len 24576
76712024-02-06T22:18:55.016Z18/30 Read block 198 len 20480
76722024-02-06T22:18:55.016Z19/30 Read block 123 len 28672
76732024-02-06T22:18:55.016Z20/30 Write block 272 len 36864 data: 5 3 4 6 7 6 7 9 9
76742024-02-06T22:18:55.016Z21/30 Read block 192 len 28672
76752024-02-06T22:18:55.020Z22/30 Flush
76762024-02-06T22:18:55.023Z23/30 Write block 420 len 4096 data: 9
76772024-02-06T22:18:55.023Z24/30 Flush
76782024-02-06T22:18:55.024Z25/30 Write block 213 len 24576 data: 7 5 5 4 5 6
76792024-02-06T22:18:55.024Z26/30 Read block 376 len 8192
76802024-02-06T22:18:55.028Z27/30 Write block 213 len 8192 data: 8 6
76812024-02-06T22:18:55.028Z28/30 Write block 382 len 28672 data: 6 6 6 7 7 5 4
76822024-02-06T22:18:55.028Z29/30 Write block 196 len 8192 data: 8 7
76832024-02-06T22:18:55.028Z30/30 Read block 456 len 40960
76842024-02-06T22:18:55.028Z----------------------------------------------------------------
76852024-02-06T22:18:55.028Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
76862024-02-06T22:18:55.029ZStates: Active Active Active
76872024-02-06T22:18:55.029ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76882024-02-06T22:18:55.029Z 1 Acked 1000 Write 2 Done Done Sent false
76892024-02-06T22:18:55.029Z 2 Acked 1001 Read 5 Done Done Sent false
76902024-02-06T22:18:55.029Z 3 Acked 1002 Write 4 Done Done Sent false
76912024-02-06T22:18:55.029Z 4 Acked 1003 Write 9 Done Done Sent false
76922024-02-06T22:18:55.029Z 5 Acked 1004 Write 8 Done Done Sent false
76932024-02-06T22:18:55.029Z 6 Acked 1005 Read 3 Done Done Sent false
76942024-02-06T22:18:55.029Z 7 Acked 1006 Write 9 Done Done Sent false
76952024-02-06T22:18:55.030Z 8 Acked 1007 Read 3 Done Done Sent false
76962024-02-06T22:18:55.030Z 9 Acked 1008 Write 4 Done Done Sent false
76972024-02-06T22:18:55.030Z 10 Acked 1009 Read 8 Done Done Sent false
76982024-02-06T22:18:55.030Z 11 Acked 1010 Write 6 Done Done Sent false
76992024-02-06T22:18:55.030Z 12 Acked 1011 Read 8 Done Done Sent false
77002024-02-06T22:18:55.030Z 13 Acked 1012 Write 9 Done Done Sent false
77012024-02-06T22:18:55.030Z 14 Acked 1013 Read 5 Done Done Sent false
77022024-02-06T22:18:55.030Z 15 Acked 1014 Write 6 Done Done Sent false
77032024-02-06T22:18:55.030Z 16 Acked 1015 Read 4 Done Done Sent false
77042024-02-06T22:18:55.030Z 17 Acked 1016 Read 6 Done Done Sent false
77052024-02-06T22:18:55.030Z 18 Acked 1017 Read 5 Done Done Sent false
77062024-02-06T22:18:55.030Z 19 Acked 1018 Read 7 Done Done Sent false
77072024-02-06T22:18:55.031Z 20 Acked 1019 Write 9 Done Done Sent false
77082024-02-06T22:18:55.031Z 21 Acked 1020 Read 7 Done Done Sent false
77092024-02-06T22:18:55.031Z 22 Acked 1021 Flush 0 Done Done Sent false
77102024-02-06T22:18:55.031Z 23 Acked 1022 Write 1 Done Done Sent false
77112024-02-06T22:18:55.031Z 24 Acked 1023 Flush 0 Done Done Sent false
77122024-02-06T22:18:55.031Z 25 Acked 1024 Write 6 Done Done Sent false
77132024-02-06T22:18:55.031Z 26 Acked 1025 Read 2 Done Done Sent false
77142024-02-06T22:18:55.031Z 27 Acked 1026 Write 2 Done Done Sent false
77152024-02-06T22:18:55.031Z 28 Acked 1027 Write 7 Done Done Sent false
77162024-02-06T22:18:55.031Z 29 Acked 1028 Write 2 Done Done Sent false
77172024-02-06T22:18:55.031Z 30 Acked 1029 Read 10 Done Sent Sent false
77182024-02-06T22:18:55.032Z STATES DS:0 DS:1 DS:2 TOTAL
77192024-02-06T22:18:55.032Z New 0 0 0 0
77202024-02-06T22:18:55.032Z Sent 0 1 30 31
77212024-02-06T22:18:55.032Z Done 30 29 0 59
77222024-02-06T22:18:55.032Z Skipped 0 0 0 0
77232024-02-06T22:18:55.032Z Error 0 0 0 0
77242024-02-06T22:18:55.032ZLast Flush: 1023 1023 0
77252024-02-06T22:18:55.032ZDownstairs last five completed:
77262024-02-06T22:18:55.036ZUpstairs last five completed: 30 29 28 27 26
77272024-02-06T22:18:55.037Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:54.244893786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1415}
77282024-02-06T22:18:55.037ZWrote out file "/tmp/test_repair_verify.data"
77292024-02-06T22:18:55.037Z
77302024-02-06T22:18:55.037ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
77312024-02-06T22:18:55.054ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77322024-02-06T22:18:55.054Z 0 000-019 66 66 66 123 123 123 F F F
77332024-02-06T22:18:55.054Z 1 020-039 68 68 68 128 128 128 F F F
77342024-02-06T22:18:55.054Z 2 040-059 68 68 68 129 129 129 F F F
77352024-02-06T22:18:55.054Z 3 060-079 64 64 64 120 120 120 F F F
77362024-02-06T22:18:55.055Z 4 080-099 68 68 68 128 128 128 F F F
77372024-02-06T22:18:55.055Z 5 100-119 68 68 68 128 128 128 F F F
77382024-02-06T22:18:55.060Z 6 120-139 70 70 69 131 131 130 F F F <---
77392024-02-06T22:18:55.060Z 7 140-159 70 70 67 131 131 126 F F F <---
77402024-02-06T22:18:55.060Z 8 160-179 68 68 68 127 127 127 F F F
77412024-02-06T22:18:55.061Z 9 180-199 64 64 64 120 120 120 T T F <---
77422024-02-06T22:18:55.061Z 10 200-219 62 62 62 118 118 118 T T F <---
77432024-02-06T22:18:55.061Z 11 220-239 70 70 67 131 131 126 F F F <---
77442024-02-06T22:18:55.061Z 12 240-259 64 64 64 120 120 120 F F F
77452024-02-06T22:18:55.061Z 13 260-279 70 70 69 131 131 130 F F F <---
77462024-02-06T22:18:55.061Z 14 280-299 70 70 69 131 131 130 F F F <---
77472024-02-06T22:18:55.061Z 15 300-319 63 63 63 119 119 119 F F F
77482024-02-06T22:18:55.061Z 16 320-339 62 62 62 117 117 117 F F F
77492024-02-06T22:18:55.061Z 17 340-359 67 67 67 126 126 126 F F F
77502024-02-06T22:18:55.062Z 18 360-379 67 67 67 126 126 126 F F F
77512024-02-06T22:18:55.062Z 19 380-399 60 60 60 115 115 115 T T F <---
77522024-02-06T22:18:55.062Z 20 400-419 70 70 68 131 131 129 F F F <---
77532024-02-06T22:18:55.062Z 21 420-439 70 70 66 132 132 125 F F F <---
77542024-02-06T22:18:55.062Z 22 440-459 58 58 58 113 113 113 F F F
77552024-02-06T22:18:55.062Z 23 460-479 58 58 58 113 113 113 F F F
77562024-02-06T22:18:55.062Z 24 480-499 63 63 63 119 119 119 F F F
77572024-02-06T22:18:55.062Z 25 500-519 70 70 68 131 131 129 F F F <---
77582024-02-06T22:18:55.062Z 26 520-539 66 66 66 125 125 125 F F F
77592024-02-06T22:18:55.063Z 27 540-559 68 68 68 128 128 128 F F F
77602024-02-06T22:18:55.063Z 28 560-579 70 70 68 131 131 128 F F F <---
77612024-02-06T22:18:55.063Z 29 580-599 66 66 66 121 121 121 F F F
77622024-02-06T22:18:55.063ZMax gen: 70, Max flush: 132
77632024-02-06T22:18:55.063ZError: Difference in extent metadata found!
77642024-02-06T22:18:55.063ZA Difference in extent metadata is expected here
77652024-02-06T22:18:55.063ZOn loop 35
77662024-02-06T22:18:55.063Z
77672024-02-06T22:18:55.063ZVerifying data now
77682024-02-06T22:18:59.655ZNo change after verify
77692024-02-06T22:18:59.656ZLoop: 35 Downstairs dump after verify (and repair):
77702024-02-06T22:18:59.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77712024-02-06T22:18:59.671Z 0 000-019 66 66 66 123 123 123 F F F
77722024-02-06T22:18:59.671Z 1 020-039 68 68 68 128 128 128 F F F
77732024-02-06T22:18:59.671Z 2 040-059 68 68 68 129 129 129 F F F
77742024-02-06T22:18:59.671Z 3 060-079 64 64 64 120 120 120 F F F
77752024-02-06T22:18:59.672Z 4 080-099 68 68 68 128 128 128 F F F
77762024-02-06T22:18:59.672Z 5 100-119 68 68 68 128 128 128 F F F
77772024-02-06T22:18:59.672Z 6 120-139 70 70 70 131 131 131 F F F
77782024-02-06T22:18:59.672Z 7 140-159 70 70 70 131 131 131 F F F
77792024-02-06T22:18:59.672Z 8 160-179 68 68 68 127 127 127 F F F
77802024-02-06T22:18:59.672Z 9 180-199 71 71 71 133 133 133 F F F
77812024-02-06T22:18:59.672Z 10 200-219 71 71 71 133 133 133 F F F
77822024-02-06T22:18:59.672Z 11 220-239 70 70 70 131 131 131 F F F
77832024-02-06T22:18:59.678Z 12 240-259 64 64 64 120 120 120 F F F
77842024-02-06T22:18:59.678Z 13 260-279 70 70 70 131 131 131 F F F
77852024-02-06T22:18:59.678Z 14 280-299 70 70 70 131 131 131 F F F
77862024-02-06T22:18:59.679Z 15 300-319 63 63 63 119 119 119 F F F
77872024-02-06T22:18:59.679Z 16 320-339 62 62 62 117 117 117 F F F
77882024-02-06T22:18:59.679Z 17 340-359 67 67 67 126 126 126 F F F
77892024-02-06T22:18:59.679Z 18 360-379 67 67 67 126 126 126 F F F
77902024-02-06T22:18:59.679Z 19 380-399 71 71 71 133 133 133 F F F
77912024-02-06T22:18:59.679Z 20 400-419 70 70 70 131 131 131 F F F
77922024-02-06T22:18:59.679Z 21 420-439 70 70 70 132 132 132 F F F
77932024-02-06T22:18:59.679Z 22 440-459 58 58 58 113 113 113 F F F
77942024-02-06T22:18:59.679Z 23 460-479 58 58 58 113 113 113 F F F
77952024-02-06T22:18:59.679Z 24 480-499 63 63 63 119 119 119 F F F
77962024-02-06T22:18:59.680Z 25 500-519 70 70 70 131 131 131 F F F
77972024-02-06T22:18:59.680Z 26 520-539 66 66 66 125 125 125 F F F
77982024-02-06T22:18:59.680Z 27 540-559 68 68 68 128 128 128 F F F
77992024-02-06T22:18:59.680Z 28 560-579 70 70 70 131 131 131 F F F
78002024-02-06T22:18:59.680Z 29 580-599 66 66 66 121 121 121 F F F
78012024-02-06T22:18:59.680ZMax gen: 71, Max flush: 133
78022024-02-06T22:18:59.680Z
78032024-02-06T22:18:59.680ZBegin loop 36
78042024-02-06T22:18:59.680ZDownstairs to restart: 0
78052024-02-06T22:18:59.693Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.908605062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423}
78062024-02-06T22:18:59.693Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\nCrucible runtime is spawned
78072024-02-06T22:18:59.700Z cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.908941447Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423}
78082024-02-06T22:18:59.700Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.908999349Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423}
78092024-02-06T22:18:59.700Z{"msg":"Crucible 860521d8-c96a-439f-b490-d0c05532f184 has session id: 2eed9648-886a-4f34-8ac2-587e9152c386","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909043473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78102024-02-06T22:18:59.700Z{"msg":"Upstairs opts: Upstairs UUID: 860521d8-c96a-439f-b490-d0c05532f184, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909100585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78112024-02-06T22:18:59.700Z{"msg":"Crucible stats registered with UUID: 860521d8-c96a-439f-b490-d0c05532f184","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909143749Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78122024-02-06T22:18:59.700Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909187304Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423}
78132024-02-06T22:18:59.700Z{"msg":"860521d8-c96a-439f-b490-d0c05532f184 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.9092156Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78142024-02-06T22:18:59.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909258044Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78152024-02-06T22:18:59.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.90928993Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78162024-02-06T22:18:59.701Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909336143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78172024-02-06T22:18:59.701Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909381207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78182024-02-06T22:18:59.701Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909417902Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78192024-02-06T22:18:59.701Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909444679Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78202024-02-06T22:18:59.701Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909479744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78212024-02-06T22:18:59.701Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909607957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78222024-02-06T22:18:59.701Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909679617Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78232024-02-06T22:18:59.701Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.90973Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78242024-02-06T22:18:59.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909757147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78252024-02-06T22:18:59.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.909800181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78262024-02-06T22:18:59.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.910188618Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78272024-02-06T22:18:59.702Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.910237402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78282024-02-06T22:18:59.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.910267627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78292024-02-06T22:18:59.702Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.910301763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78302024-02-06T22:18:59.702Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.910739834Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78312024-02-06T22:18:59.702Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.910885084Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78322024-02-06T22:18:59.702Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911353181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78332024-02-06T22:18:59.702Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911391906Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78342024-02-06T22:18:59.702Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911417772Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78352024-02-06T22:18:59.702Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911445948Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78362024-02-06T22:18:59.702Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911548794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78372024-02-06T22:18:59.702Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911568382Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78382024-02-06T22:18:59.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911590029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78392024-02-06T22:18:59.702Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.911604567Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78402024-02-06T22:18:59.702Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912038298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78412024-02-06T22:18:59.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912463231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78422024-02-06T22:18:59.703Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912504345Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78432024-02-06T22:18:59.703Z{"msg":"[0]R flush_numbers[0..12]: [123, 128, 129, 120, 128, 128, 131, 131, 127, 133, 133, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912522443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78442024-02-06T22:18:59.703Z{"msg":"[0]R generation[0..12]: [66, 68, 68, 64, 68, 68, 70, 70, 68, 71, 71, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.91254421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78452024-02-06T22:18:59.703Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912559578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78462024-02-06T22:18:59.703Z{"msg":"[1]R flush_numbers[0..12]: [123, 128, 129, 120, 128, 128, 131, 131, 127, 133, 133, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912580625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78472024-02-06T22:18:59.703Z{"msg":"[1]R generation[0..12]: [66, 68, 68, 64, 68, 68, 70, 70, 68, 71, 71, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912594883Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78482024-02-06T22:18:59.703Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.91261625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78492024-02-06T22:18:59.703Z{"msg":"[2]R flush_numbers[0..12]: [123, 128, 129, 120, 128, 128, 131, 131, 127, 133, 133, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912630978Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78502024-02-06T22:18:59.703Z{"msg":"[2]R generation[0..12]: [66, 68, 68, 64, 68, 68, 70, 70, 68, 71, 71, 70]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
78512024-02-06T22:18:59.703Z,"time":"2024-02-06T22:18:58.912652005Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78522024-02-06T22:18:59.703Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912670053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78532024-02-06T22:18:59.703Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912683841Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78542024-02-06T22:18:59.703Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912698429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78552024-02-06T22:18:59.703Z{"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912711457Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78562024-02-06T22:18:59.703Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912733614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78572024-02-06T22:18:59.703Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912747563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78582024-02-06T22:18:59.703Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912774609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78592024-02-06T22:18:59.703Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912789827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78602024-02-06T22:18:59.703Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912810024Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78612024-02-06T22:18:59.703Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912823802Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78622024-02-06T22:18:59.703Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.91284261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78632024-02-06T22:18:59.703Z{"msg":"860521d8-c96a-439f-b490-d0c05532f184 is now active with session: 2eed9648-886a-4f34-8ac2-587e9152c386","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912856528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78642024-02-06T22:18:59.703Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912881854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423,"session_id":"2eed9648-886a-4f34-8ac2-587e9152c386"}
78652024-02-06T22:18:59.704Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:18:58.912894903Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423}
78662024-02-06T22:18:59.707ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78672024-02-06T22:18:59.708ZLoading write count information from file "/tmp/test_repair_verify.data"
78682024-02-06T22:18:59.708ZRun Repair workload
78692024-02-06T22:18:59.708Z01/30 Write block 209 len 28672 data: 11 10 8 7 9 7 6
78702024-02-06T22:18:59.708Z02/30 Read block 10 len 32768
78712024-02-06T22:18:59.708Z03/30 Write block 480 len 20480 data: 3 3 4 5 4
78722024-02-06T22:18:59.708Z04/30 Write block 123 len 4096 data: 9
78732024-02-06T22:18:59.708Z05/30 Read block 493 len 8192
78742024-02-06T22:18:59.708Z06/30 Flush
78752024-02-06T22:18:59.708Z07/30 Write block 213 len 16384 data: 10 8 7 5
78762024-02-06T22:18:59.709Z08/30 Write block 564 len 8192 data: 7 9
78772024-02-06T22:18:59.709Z09/30 Flush
78782024-02-06T22:18:59.709Z10/30 Write block 261 len 8192 data: 6 6
78792024-02-06T22:18:59.709Z11/30 Read block 198 len 36864
78802024-02-06T22:18:59.713Z12/30 Read block 508 len 32768
78812024-02-06T22:18:59.713Z13/30 Read block 159 len 32768
78822024-02-06T22:18:59.713Z14/30 Read block 364 len 8192
78832024-02-06T22:18:59.713Z15/30 Read block 440 len 32768
78842024-02-06T22:18:59.713Z16/30 Write block 482 len 40960 data: 5 6 5 5 5 4 3 4 3 4
78852024-02-06T22:18:59.717Z17/30 Read block 233 len 16384
78862024-02-06T22:18:59.717Z18/30 Read block 387 len 16384
78872024-02-06T22:18:59.717Z19/30 Read block 327 len 20480
78882024-02-06T22:18:59.717Z20/30 Write block 82 len 40960 data: 5 5 5 4 4 4 4 4 4 5
78892024-02-06T22:18:59.721Z21/30 Write block 128 len 16384 data: 10 7 8 7
78902024-02-06T22:18:59.721Z22/30 Read block 403 len 32768
78912024-02-06T22:18:59.721Z23/30 Flush
78922024-02-06T22:18:59.722Z24/30 Write block 10 len 16384 data: 7 4 4 3
78932024-02-06T22:18:59.722Z25/30 Flush
78942024-02-06T22:18:59.725Z26/30 Write block 433 len 28672 data: 4 5 5 7 7 6 6
78952024-02-06T22:18:59.725Z27/30 Write block 179 len 36864 data: 11 11 11 9 10 12 12 12 11
78962024-02-06T22:18:59.725Z28/30 Read block 216 len 12288
78972024-02-06T22:18:59.729Z29/30 Write block 435 len 16384 data: 6 8 8 7
78982024-02-06T22:18:59.729Z30/30 Read block 337 len 16384
78992024-02-06T22:18:59.729Z----------------------------------------------------------------
79002024-02-06T22:18:59.729Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
79012024-02-06T22:18:59.729ZStates: Active Active Active
79022024-02-06T22:18:59.729ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
79032024-02-06T22:18:59.729Z 1 Acked 1000 Write 7 Done Done Done false
79042024-02-06T22:18:59.730Z 2 Acked 1001 Read 8 Sent Done Done false
79052024-02-06T22:18:59.730Z 3 Acked 1002 Write 5 Sent Done Done false
79062024-02-06T22:18:59.730Z 4 Acked 1003 Write 1 Sent Done Done false
79072024-02-06T22:18:59.730Z 5 Acked 1004 Read 2 Sent Done Done false
79082024-02-06T22:18:59.730Z 6 Acked 1005 Flush 0 Sent Done Done false
79092024-02-06T22:18:59.730Z 7 Acked 1006 Write 4 Sent Done Done false
79102024-02-06T22:18:59.730Z 8 Acked 1007 Write 2 Sent Done Done false
79112024-02-06T22:18:59.730Z 9 Acked 1008 Flush 0 Sent Done Done false
79122024-02-06T22:18:59.730Z 10 Acked 1009 Write 2 Sent Done Done false
79132024-02-06T22:18:59.730Z 11 Acked 1010 Read 9 Sent Done Done false
79142024-02-06T22:18:59.730Z 12 Acked 1011 Read 8 Sent Done Done false
79152024-02-06T22:18:59.731Z 13 Acked 1012 Read 8 Sent Done Done false
79162024-02-06T22:18:59.731Z 14 Acked 1013 Read 2 Sent Done Done false
79172024-02-06T22:18:59.731Z 15 Acked 1014 Read 8 Sent Done Done false
79182024-02-06T22:18:59.731Z 16 Acked 1015 Write 10 Sent Done Done false
79192024-02-06T22:18:59.731Z 17 Acked 1016 Read 4 Sent Done Done false
79202024-02-06T22:18:59.731Z 18 Acked 1017 Read 4 Sent Done Done false
79212024-02-06T22:18:59.731Z 19 Acked 1018 Read 5 Sent Done Done false
79222024-02-06T22:18:59.731Z 20 Acked 1019 Write 10 Sent Done Done false
79232024-02-06T22:18:59.731Z 21 Acked 1020 Write 4 Sent Done Done false
79242024-02-06T22:18:59.731Z 22 Acked 1021 Read 8 Sent Done Done false
79252024-02-06T22:18:59.731Z 23 Acked 1022 Flush 0 Sent Done Done false
79262024-02-06T22:18:59.731Z 24 Acked 1023 Write 4 Sent Done Done false
79272024-02-06T22:18:59.732Z 25 Acked 1024 Flush 0 Sent Done Done false
79282024-02-06T22:18:59.732Z 26 Acked 1025 Write 7 Sent Done Done false
79292024-02-06T22:18:59.732Z 27 Acked 1026 Write 9 Sent Done Done false
79302024-02-06T22:18:59.732Z 28 Acked 1027 Read 3 Sent Done Done false
79312024-02-06T22:18:59.732Z 29 Acked 1028 Write 4 Sent Done Done false
79322024-02-06T22:18:59.732Z 30 Acked 1029 Read 4 Sent Done Done false
79332024-02-06T22:18:59.732Z STATES DS:0 DS:1 DS:2 TOTAL
79342024-02-06T22:18:59.732Z New 0 0 0 0
79352024-02-06T22:18:59.732Z Sent 29 0 0 29
79362024-02-06T22:18:59.732Z Done 1 30 30 61
79372024-02-06T22:18:59.732Z Skipped 0 0 0 0
79382024-02-06T22:18:59.733Z Error 0 0 0 0
79392024-02-06T22:18:59.733ZLast Flush: 0 1024 1024
79402024-02-06T22:18:59.733ZDownstairs last five completed:
79412024-02-06T22:18:59.733ZUpstairs last five completed: 30 29 28 27 26
79422024-02-06T22:18:59.733Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:18:58.943745894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1423}
79432024-02-06T22:18:59.733ZWrote out file "/tmp/test_repair_verify.data"
79442024-02-06T22:18:59.742Z
79452024-02-06T22:18:59.742ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
79462024-02-06T22:18:59.750ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79472024-02-06T22:18:59.750Z 0 000-019 66 72 72 123 137 137 F F F <---
79482024-02-06T22:18:59.751Z 1 020-039 68 68 68 128 128 128 F F F
79492024-02-06T22:18:59.751Z 2 040-059 68 68 68 129 129 129 F F F
79502024-02-06T22:18:59.751Z 3 060-079 64 64 64 120 120 120 F F F
79512024-02-06T22:18:59.751Z 4 080-099 68 72 72 128 136 136 F F F <---
79522024-02-06T22:18:59.751Z 5 100-119 68 68 68 128 128 128 F F F
79532024-02-06T22:18:59.751Z 6 120-139 70 72 72 131 136 136 F F F <---
79542024-02-06T22:18:59.751Z 7 140-159 70 70 70 131 131 131 F F F
79552024-02-06T22:18:59.751Z 8 160-179 68 68 68 127 127 127 F T T <---
79562024-02-06T22:18:59.751Z 9 180-199 71 71 71 133 133 133 F T T <---
79572024-02-06T22:18:59.751Z 10 200-219 71 72 72 133 135 135 T F F <---
79582024-02-06T22:18:59.751Z 11 220-239 70 70 70 131 131 131 F F F
79592024-02-06T22:18:59.752Z 12 240-259 64 64 64 120 120 120 F F F
79602024-02-06T22:18:59.752Z 13 260-279 70 72 72 131 136 136 F F F <---
79612024-02-06T22:18:59.756Z 14 280-299 70 70 70 131 131 131 F F F
79622024-02-06T22:18:59.756Z 15 300-319 63 63 63 119 119 119 F F F
79632024-02-06T22:18:59.756Z 16 320-339 62 62 62 117 117 117 F F F
79642024-02-06T22:18:59.757Z 17 340-359 67 67 67 126 126 126 F F F
79652024-02-06T22:18:59.757Z 18 360-379 67 67 67 126 126 126 F F F
79662024-02-06T22:18:59.757Z 19 380-399 71 71 71 133 133 133 F F F
79672024-02-06T22:18:59.757Z 20 400-419 70 70 70 131 131 131 F F F
79682024-02-06T22:18:59.757Z 21 420-439 70 70 70 132 132 132 F T T <---
79692024-02-06T22:18:59.757Z 22 440-459 58 58 58 113 113 113 F F F
79702024-02-06T22:18:59.757Z 23 460-479 58 58 58 113 113 113 F F F
79712024-02-06T22:18:59.757Z 24 480-499 63 72 72 119 136 136 F F F <---
79722024-02-06T22:18:59.757Z 25 500-519 70 70 70 131 131 131 F F F
79732024-02-06T22:18:59.757Z 26 520-539 66 66 66 125 125 125 F F F
79742024-02-06T22:18:59.757Z 27 540-559 68 68 68 128 128 128 F F F
79752024-02-06T22:18:59.758Z 28 560-579 70 72 72 131 135 135 F F F <---
79762024-02-06T22:18:59.758Z 29 580-599 66 66 66 121 121 121 F F F
79772024-02-06T22:18:59.758ZMax gen: 72, Max flush: 137
79782024-02-06T22:18:59.758ZError: Difference in extent metadata found!
79792024-02-06T22:18:59.758ZA Difference in extent metadata is expected here
79802024-02-06T22:18:59.758ZOn loop 36
79812024-02-06T22:18:59.758Z
79822024-02-06T22:18:59.762ZVerifying data now
79832024-02-06T22:19:05.284ZNo change after verify
79842024-02-06T22:19:05.284ZLoop: 36 Downstairs dump after verify (and repair):
79852024-02-06T22:19:05.299ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79862024-02-06T22:19:05.299Z 0 000-019 72 72 72 137 137 137 F F F
79872024-02-06T22:19:05.299Z 1 020-039 68 68 68 128 128 128 F F F
79882024-02-06T22:19:05.299Z 2 040-059 68 68 68 129 129 129 F F F
79892024-02-06T22:19:05.299Z 3 060-079 64 64 64 120 120 120 F F F
79902024-02-06T22:19:05.299Z 4 080-099 72 72 72 136 136 136 F F F
79912024-02-06T22:19:05.305Z 5 100-119 68 68 68 128 128 128 F F F
79922024-02-06T22:19:05.305Z 6 120-139 72 72 72 136 136 136 F F F
79932024-02-06T22:19:05.305Z 7 140-159 70 70 70 131 131 131 F F F
79942024-02-06T22:19:05.305Z 8 160-179 73 73 73 138 138 138 F F F
79952024-02-06T22:19:05.305Z 9 180-199 73 73 73 138 138 138 F F F
79962024-02-06T22:19:05.305Z 10 200-219 72 72 72 135 135 135 F F F
79972024-02-06T22:19:05.306Z 11 220-239 70 70 70 131 131 131 F F F
79982024-02-06T22:19:05.306Z 12 240-259 64 64 64 120 120 120 F F F
79992024-02-06T22:19:05.306Z 13 260-279 72 72 72 136 136 136 F F F
80002024-02-06T22:19:05.306Z 14 280-299 70 70 70 131 131 131 F F F
80012024-02-06T22:19:05.306Z 15 300-319 63 63 63 119 119 119 F F F
80022024-02-06T22:19:05.306Z 16 320-339 62 62 62 117 117 117 F F F
80032024-02-06T22:19:05.306Z 17 340-359 67 67 67 126 126 126 F F F
80042024-02-06T22:19:05.306Z 18 360-379 67 67 67 126 126 126 F F F
80052024-02-06T22:19:05.307Z 19 380-399 71 71 71 133 133 133 F F F
80062024-02-06T22:19:05.307Z 20 400-419 70 70 70 131 131 131 F F F
80072024-02-06T22:19:05.307Z 21 420-439 73 73 73 138 138 138 F F F
80082024-02-06T22:19:05.307Z 22 440-459 58 58 58 113 113 113 F F F
80092024-02-06T22:19:05.307Z 23 460-479 58 58 58 113 113 113 F F F
80102024-02-06T22:19:05.307Z 24 480-499 72 72 72 136 136 136 F F F
80112024-02-06T22:19:05.307Z 25 500-519 70 70 70 131 131 131 F F F
80122024-02-06T22:19:05.308Z 26 520-539 66 66 66 125 125 125 F F F
80132024-02-06T22:19:05.308Z 27 540-559 68 68 68 128 128 128 F F F
80142024-02-06T22:19:05.308Z 28 560-579 72 72 72 135 135 135 F F F
80152024-02-06T22:19:05.308Z 29 580-599 66 66 66 121 121 121 F F F
80162024-02-06T22:19:05.308ZMax gen: 73, Max flush: 138
80172024-02-06T22:19:05.308Z
80182024-02-06T22:19:05.308ZBegin loop 37
80192024-02-06T22:19:05.309ZDownstairs to restart: 1
80202024-02-06T22:19:05.321Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.536384662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431}
80212024-02-06T22:19:05.328Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.53669552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431}
80222024-02-06T22:19:05.328Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.536740054Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431}
80232024-02-06T22:19:05.328Z{"msg":"Crucible a577be94-a1cb-4170-860b-d7e5daf2ee88 has session id: 9bc84fb1-167d-49a7-b018-d5731c36d2a5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.536763361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431Crucible runtime is spawned
80242024-02-06T22:19:05.329Z,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80252024-02-06T22:19:05.329Z{"msg":"Upstairs opts: Upstairs UUID: a577be94-a1cb-4170-860b-d7e5daf2ee88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.536816753Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80262024-02-06T22:19:05.329Z{"msg":"Crucible stats registered with UUID: a577be94-a1cb-4170-860b-d7e5daf2ee88","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.53684366Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80272024-02-06T22:19:05.329Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.536875975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431}
80282024-02-06T22:19:05.329Z{"msg":"a577be94-a1cb-4170-860b-d7e5daf2ee88 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.536950355Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80292024-02-06T22:19:05.329Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.53698897Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80302024-02-06T22:19:05.329Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537024505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80312024-02-06T22:19:05.329Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537059441Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80322024-02-06T22:19:05.329Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537081498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80332024-02-06T22:19:05.329Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537112524Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80342024-02-06T22:19:05.330Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537217499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80352024-02-06T22:19:05.330Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537266273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80362024-02-06T22:19:05.330Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537299648Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80372024-02-06T22:19:05.330Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537335663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80382024-02-06T22:19:05.330Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.53736106Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80392024-02-06T22:19:05.330Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537404554Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80402024-02-06T22:19:05.330Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.537422302Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80412024-02-06T22:19:05.330Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.538012492Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80422024-02-06T22:19:05.330Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.538073234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80432024-02-06T22:19:05.331Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.538127506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80442024-02-06T22:19:05.331Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.538178539Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80452024-02-06T22:19:05.331Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.538641307Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80462024-02-06T22:19:05.331Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.538783918Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80472024-02-06T22:19:05.331Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539030644Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80482024-02-06T22:19:05.331Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539078968Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80492024-02-06T22:19:05.331Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539168726Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80502024-02-06T22:19:05.331Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.5392105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80512024-02-06T22:19:05.331Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539237776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80522024-02-06T22:19:05.331Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539265482Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80532024-02-06T22:19:05.332Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539290819Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80542024-02-06T22:19:05.332Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539323315Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80552024-02-06T22:19:05.332Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.539622994Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80562024-02-06T22:19:05.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540087302Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80572024-02-06T22:19:05.332Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540135175Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80582024-02-06T22:19:05.332Z{"msg":"[0]R flush_numbers[0..12]: [137, 128, 129, 120, 136, 128, 136, 131, 138, 138, 135, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.54017195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80592024-02-06T22:19:05.332Z{"msg":"[0]R generation[0..12]: [72, 68, 68, 64, 72, 68, 72, 70, 73, 73, 72, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540198856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80602024-02-06T22:19:05.332Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540214974Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80612024-02-06T22:19:05.332Z{"msg":"[1]R flush_numbers[0..12]: [137, 128, 129, 120, 136, 128, 136, 131, 138, 138, 135, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540237781Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80622024-02-06T22:19:05.332Z{"msg":"[1]R generation[0..12]: [72, 68, 68, 64, 72, 68, 72, 70, 73, 73, 72, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540253049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80632024-02-06T22:19:05.333Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
80642024-02-06T22:19:05.333Z,"time":"2024-02-06T22:19:04.540275506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80652024-02-06T22:19:05.333Z{"msg":"[2]R flush_numbers[0..12]: [137, 128, 129, 120, 136, 128, 136, 131, 138, 138, 135, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540291814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80662024-02-06T22:19:05.333Z{"msg":"[2]R generation[0..12]: [72, 68, 68, 64, 72, 68, 72, 70, 73, 73, 72, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540313831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80672024-02-06T22:19:05.333Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540329049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80682024-02-06T22:19:05.333Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540356405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80692024-02-06T22:19:05.333Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540371113Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80702024-02-06T22:19:05.333Z{"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.54039222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80712024-02-06T22:19:05.333Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540405779Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80722024-02-06T22:19:05.333Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540427356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80732024-02-06T22:19:05.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540440984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80742024-02-06T22:19:05.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540461621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80752024-02-06T22:19:05.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540475809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80762024-02-06T22:19:05.334Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540495736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80772024-02-06T22:19:05.334Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540509225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80782024-02-06T22:19:05.334Z{"msg":"a577be94-a1cb-4170-860b-d7e5daf2ee88 is now active with session: 9bc84fb1-167d-49a7-b018-d5731c36d2a5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540523583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80792024-02-06T22:19:05.334Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.54054388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431,"session_id":"9bc84fb1-167d-49a7-b018-d5731c36d2a5"}
80802024-02-06T22:19:05.334Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:04.540557208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431}
80812024-02-06T22:19:05.339ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80822024-02-06T22:19:05.339ZLoading write count information from file "/tmp/test_repair_verify.data"
80832024-02-06T22:19:05.339ZRun Repair workload
80842024-02-06T22:19:05.339Z01/30 Write block 400 len 32768 data: 7 8 8 6 5 4 5 4
80852024-02-06T22:19:05.339Z02/30 Read block 255 len 16384
80862024-02-06T22:19:05.339Z03/30 Read block 180 len 24576
80872024-02-06T22:19:05.339Z04/30 Flush
80882024-02-06T22:19:05.340Z05/30 Flush
80892024-02-06T22:19:05.340Z06/30 Write block 529 len 12288 data: 8 9 9
80902024-02-06T22:19:05.340Z07/30 Read block 201 len 16384
80912024-02-06T22:19:05.340Z08/30 Read block 378 len 36864
80922024-02-06T22:19:05.340Z09/30 Read block 573 len 4096
80932024-02-06T22:19:05.340Z10/30 Flush
80942024-02-06T22:19:05.340Z11/30 Write block 277 len 24576 data: 7 8 10 10 9 8
80952024-02-06T22:19:05.340Z12/30 Flush
80962024-02-06T22:19:05.344Z13/30 Write block 381 len 40960 data: 8 7 7 7 8 8 6 5 4 5
80972024-02-06T22:19:05.345Z14/30 Write block 25 len 36864 data: 7 7 7 7 6 6 4 2 3
80982024-02-06T22:19:05.345Z15/30 Write block 233 len 16384 data: 6 8 8 8
80992024-02-06T22:19:05.345Z16/30 Write block 1 len 12288 data: 3 5 7
81002024-02-06T22:19:05.345Z17/30 Read block 465 len 24576
81012024-02-06T22:19:05.345Z18/30 Read block 402 len 32768
81022024-02-06T22:19:05.349Z19/30 Read block 261 len 28672
81032024-02-06T22:19:05.349Z20/30 Write block 191 len 28672 data: 5 5 5 6 6 9 8
81042024-02-06T22:19:05.349Z21/30 Write block 416 len 8192 data: 9 12
81052024-02-06T22:19:05.349Z22/30 Read block 50 len 16384
81062024-02-06T22:19:05.349Z23/30 Write block 345 len 12288 data: 7 9 10
81072024-02-06T22:19:05.349Z24/30 Write block 49 len 8192 data: 9 10
81082024-02-06T22:19:05.354Z25/30 Write block 293 len 36864 data: 9 9 10 8 6 5 5 7 6
81092024-02-06T22:19:05.354Z26/30 Write block 303 len 16384 data: 4 4 7 7
81102024-02-06T22:19:05.354Z27/30 Write block 569 len 24576 data: 7 7 10 11 9 9
81112024-02-06T22:19:05.354Z28/30 Write block 231 len 12288 data: 7 7 7
81122024-02-06T22:19:05.354Z29/30 Read block 456 len 4096
81132024-02-06T22:19:05.355Z30/30 Write block 189 len 16384 data: 6 5 6 6
81142024-02-06T22:19:05.355Z----------------------------------------------------------------
81152024-02-06T22:19:05.355Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
81162024-02-06T22:19:05.355ZStates: Active Active Active
81172024-02-06T22:19:05.355ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
81182024-02-06T22:19:05.355Z 1 Acked 1000 Write 8 Done Done Done false
81192024-02-06T22:19:05.355Z 2 Acked 1001 Read 4 Done Done Done false
81202024-02-06T22:19:05.355Z 3 Acked 1002 Read 6 Done Sent Done false
81212024-02-06T22:19:05.355Z 4 Acked 1003 Flush 0 Done Sent Done false
81222024-02-06T22:19:05.355Z 5 Acked 1004 Flush 0 Done Sent Done false
81232024-02-06T22:19:05.356Z 6 Acked 1005 Write 3 Done Sent Done false
81242024-02-06T22:19:05.356Z 7 Acked 1006 Read 4 Done Sent Done false
81252024-02-06T22:19:05.356Z 8 Acked 1007 Read 9 Done Sent Done false
81262024-02-06T22:19:05.356Z 9 Acked 1008 Read 1 Done Sent Done false
81272024-02-06T22:19:05.356Z 10 Acked 1009 Flush 0 Done Sent Done false
81282024-02-06T22:19:05.356Z 11 Acked 1010 Write 6 Done Sent Done false
81292024-02-06T22:19:05.356Z 12 Acked 1011 Flush 0 Done Sent Done false
81302024-02-06T22:19:05.356Z 13 Acked 1012 Write 10 Done Sent Done false
81312024-02-06T22:19:05.356Z 14 Acked 1013 Write 9 Done Sent Done false
81322024-02-06T22:19:05.357Z 15 Acked 1014 Write 4 Done Sent Done false
81332024-02-06T22:19:05.357Z 16 Acked 1015 Write 3 Done Sent Done false
81342024-02-06T22:19:05.357Z 17 Acked 1016 Read 6 Done Sent Done false
81352024-02-06T22:19:05.357Z 18 Acked 1017 Read 8 Done Sent Done false
81362024-02-06T22:19:05.357Z 19 Acked 1018 Read 7 Done Sent Done false
81372024-02-06T22:19:05.357Z 20 Acked 1019 Write 7 Done Sent Done false
81382024-02-06T22:19:05.357Z 21 Acked 1020 Write 2 Done Sent Done false
81392024-02-06T22:19:05.357Z 22 Acked 1021 Read 4 Done Sent Done false
81402024-02-06T22:19:05.357Z 23 Acked 1022 Write 3 Done Sent Done false
81412024-02-06T22:19:05.358Z 24 Acked 1023 Write 2 Done Sent Done false
81422024-02-06T22:19:05.358Z 25 Acked 1024 Write 9 Done Sent Done false
81432024-02-06T22:19:05.358Z 26 Acked 1025 Write 4 Done Sent Done false
81442024-02-06T22:19:05.358Z 27 Acked 1026 Write 6 Done Sent Done false
81452024-02-06T22:19:05.358Z 28 Acked 1027 Write 3 Done Sent Done false
81462024-02-06T22:19:05.358Z 29 Acked 1028 Read 1 Done Sent Done false
81472024-02-06T22:19:05.358Z 30 Acked 1029 Write 4 Sent Sent Sent false
81482024-02-06T22:19:05.358Z STATES DS:0 DS:1 DS:2 TOTAL
81492024-02-06T22:19:05.359Z New 0 0 0 0
81502024-02-06T22:19:05.359Z Sent 1 28 1 30
81512024-02-06T22:19:05.359Z Done 29 2 29 60
81522024-02-06T22:19:05.359Z Skipped 0 0 0 0
81532024-02-06T22:19:05.359Z Error 0 0 0 0
81542024-02-06T22:19:05.359ZLast Flush: 1011 0 1011
81552024-02-06T22:19:05.359ZDownstairs last five completed:
81562024-02-06T22:19:05.359ZUpstairs last five completed: 30 29 28 27 26
81572024-02-06T22:19:05.359Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:04.568757777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1431}
81582024-02-06T22:19:05.365ZWrote out file "/tmp/test_repair_verify.data"
81592024-02-06T22:19:05.365Z
81602024-02-06T22:19:05.365ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81612024-02-06T22:19:05.381ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81622024-02-06T22:19:05.381Z 0 000-019 72 72 72 137 137 137 T F T <---
81632024-02-06T22:19:05.381Z 1 020-039 68 68 68 128 128 128 T F T <---
81642024-02-06T22:19:05.381Z 2 040-059 68 68 68 129 129 129 T F T <---
81652024-02-06T22:19:05.381Z 3 060-079 64 64 64 120 120 120 F F F
81662024-02-06T22:19:05.381Z 4 080-099 72 72 72 136 136 136 F F F
81672024-02-06T22:19:05.381Z 5 100-119 68 68 68 128 128 128 F F F
81682024-02-06T22:19:05.381Z 6 120-139 72 72 72 136 136 136 F F F
81692024-02-06T22:19:05.386Z 7 140-159 70 70 70 131 131 131 F F F
81702024-02-06T22:19:05.386Z 8 160-179 73 73 73 138 138 138 F F F
81712024-02-06T22:19:05.386Z 9 180-199 73 73 73 138 138 138 T F T <---
81722024-02-06T22:19:05.386Z 10 200-219 72 72 72 135 135 135 F F F
81732024-02-06T22:19:05.387Z 11 220-239 70 70 70 131 131 131 T F T <---
81742024-02-06T22:19:05.387Z 12 240-259 64 64 64 120 120 120 F F F
81752024-02-06T22:19:05.387Z 13 260-279 74 72 74 142 136 142 F F F <---
81762024-02-06T22:19:05.387Z 14 280-299 74 70 74 142 131 142 T F T <---
81772024-02-06T22:19:05.387Z 15 300-319 63 63 63 119 119 119 T F T <---
81782024-02-06T22:19:05.387Z 16 320-339 62 62 62 117 117 117 F F F
81792024-02-06T22:19:05.388Z 17 340-359 67 67 67 126 126 126 T F T <---
81802024-02-06T22:19:05.388Z 18 360-379 67 67 67 126 126 126 F F F
81812024-02-06T22:19:05.388Z 19 380-399 71 71 71 133 133 133 T F T <---
81822024-02-06T22:19:05.388Z 20 400-419 74 70 74 139 131 139 T T T <---
81832024-02-06T22:19:05.388Z 21 420-439 73 73 73 138 138 138 F F F
81842024-02-06T22:19:05.388Z 22 440-459 58 58 58 113 113 113 F F F
81852024-02-06T22:19:05.388Z 23 460-479 58 58 58 113 113 113 F F F
81862024-02-06T22:19:05.389Z 24 480-499 72 72 72 136 136 136 F F F
81872024-02-06T22:19:05.389Z 25 500-519 70 70 70 131 131 131 F F F
81882024-02-06T22:19:05.389Z 26 520-539 74 66 74 141 125 141 F F F <---
81892024-02-06T22:19:05.389Z 27 540-559 68 68 68 128 128 128 F F F
81902024-02-06T22:19:05.389Z 28 560-579 72 72 72 135 135 135 T F T <---
81912024-02-06T22:19:05.389Z 29 580-599 66 66 66 121 121 121 F F F
81922024-02-06T22:19:05.389ZMax gen: 74, Max flush: 142
81932024-02-06T22:19:05.389ZError: Difference in extent metadata found!
81942024-02-06T22:19:05.390ZA Difference in extent metadata is expected here
81952024-02-06T22:19:05.390ZOn loop 37
81962024-02-06T22:19:05.390Z
81972024-02-06T22:19:05.390ZVerifying data now
81982024-02-06T22:19:11.029ZNo change after verify
81992024-02-06T22:19:11.029ZLoop: 37 Downstairs dump after verify (and repair):
82002024-02-06T22:19:11.044ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82012024-02-06T22:19:11.044Z 0 000-019 75 75 75 143 143 143 F F F
82022024-02-06T22:19:11.044Z 1 020-039 75 75 75 143 143 143 F F F
82032024-02-06T22:19:11.044Z 2 040-059 75 75 75 143 143 143 F F F
82042024-02-06T22:19:11.045Z 3 060-079 64 64 64 120 120 120 F F F
82052024-02-06T22:19:11.045Z 4 080-099 72 72 72 136 136 136 F F F
82062024-02-06T22:19:11.045Z 5 100-119 68 68 68 128 128 128 F F F
82072024-02-06T22:19:11.045Z 6 120-139 72 72 72 136 136 136 F F F
82082024-02-06T22:19:11.045Z 7 140-159 70 70 70 131 131 131 F F F
82092024-02-06T22:19:11.045Z 8 160-179 73 73 73 138 138 138 F F F
82102024-02-06T22:19:11.045Z 9 180-199 75 75 75 143 143 143 F F F
82112024-02-06T22:19:11.048Z 10 200-219 72 72 72 135 135 135 F F F
82122024-02-06T22:19:11.048Z 11 220-239 75 75 75 143 143 143 F F F
82132024-02-06T22:19:11.048Z 12 240-259 64 64 64 120 120 120 F F F
82142024-02-06T22:19:11.048Z 13 260-279 74 74 74 142 142 142 F F F
82152024-02-06T22:19:11.048Z 14 280-299 75 75 75 143 143 143 F F F
82162024-02-06T22:19:11.048Z 15 300-319 75 75 75 143 143 143 F F F
82172024-02-06T22:19:11.048Z 16 320-339 62 62 62 117 117 117 F F F
82182024-02-06T22:19:11.048Z 17 340-359 75 75 75 143 143 143 F F F
82192024-02-06T22:19:11.048Z 18 360-379 67 67 67 126 126 126 F F F
82202024-02-06T22:19:11.048Z 19 380-399 75 75 75 143 143 143 F F F
82212024-02-06T22:19:11.048Z 20 400-419 75 75 75 143 143 143 F F F
82222024-02-06T22:19:11.049Z 21 420-439 73 73 73 138 138 138 F F F
82232024-02-06T22:19:11.049Z 22 440-459 58 58 58 113 113 113 F F F
82242024-02-06T22:19:11.049Z 23 460-479 58 58 58 113 113 113 F F F
82252024-02-06T22:19:11.049Z 24 480-499 72 72 72 136 136 136 F F F
82262024-02-06T22:19:11.049Z 25 500-519 70 70 70 131 131 131 F F F
82272024-02-06T22:19:11.049Z 26 520-539 74 74 74 141 141 141 F F F
82282024-02-06T22:19:11.049Z 27 540-559 68 68 68 128 128 128 F F F
82292024-02-06T22:19:11.049Z 28 560-579 75 75 75 143 143 143 F F F
82302024-02-06T22:19:11.049Z 29 580-599 66 66 66 121 121 121 F F F
82312024-02-06T22:19:11.049ZMax gen: 75, Max flush: 143
82322024-02-06T22:19:11.049Z
82332024-02-06T22:19:11.049ZBegin loop 38
82342024-02-06T22:19:11.049ZDownstairs to restart: 1
82352024-02-06T22:19:11.065Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280231598Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439}
82362024-02-06T22:19:11.071Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280566522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439}
82372024-02-06T22:19:11.071Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
82382024-02-06T22:19:11.071Z,"time":"2024-02-06T22:19:10.280640032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439}
82392024-02-06T22:19:11.071Z{"msg":"Crucible a1b27ccd-a468-471a-aba4-bff07b460739 has session id: a932fef4-dbdb-468c-9e55-349d8db303b0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280672228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82402024-02-06T22:19:11.071Z{"msg":"Upstairs opts: Upstairs UUID: a1b27ccd-a468-471a-aba4-bff07b460739, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.28073166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82412024-02-06T22:19:11.071Z{"msg":"Crucible stats registered with UUID: a1b27ccd-a468-471a-aba4-bff07b460739","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280767695Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82422024-02-06T22:19:11.071Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280785803Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439}
82432024-02-06T22:19:11.071Z{"msg":"a1b27ccd-a468-471a-aba4-bff07b460739 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280801121Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82442024-02-06T22:19:11.071Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280825347Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82452024-02-06T22:19:11.071Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280844205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82462024-02-06T22:19:11.071Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.28087904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82472024-02-06T22:19:11.072Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280908166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82482024-02-06T22:19:11.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280935013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82492024-02-06T22:19:11.072Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.280974957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82502024-02-06T22:19:11.072Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281008783Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82512024-02-06T22:19:11.072Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281042198Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82522024-02-06T22:19:11.072Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281069574Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82532024-02-06T22:19:11.072Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281109689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82542024-02-06T22:19:11.072Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281146024Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82552024-02-06T22:19:11.072Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:10.28124805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82562024-02-06T22:19:11.072Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:10.281302243Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82572024-02-06T22:19:11.072Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:10.281449883Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82582024-02-06T22:19:11.072Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281494837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82592024-02-06T22:19:11.072Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281534022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82602024-02-06T22:19:11.072Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.281577086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82612024-02-06T22:19:11.072Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.282013857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82622024-02-06T22:19:11.072Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.28206742Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82632024-02-06T22:19:11.072Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.282092536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82642024-02-06T22:19:11.072Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.282132221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82652024-02-06T22:19:11.072Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.28258602Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82662024-02-06T22:19:11.072Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.282722711Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82672024-02-06T22:19:11.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.283067304Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82682024-02-06T22:19:11.072Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.2831031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82692024-02-06T22:19:11.072Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.283124017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82702024-02-06T22:19:11.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.283155243Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82712024-02-06T22:19:11.073Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.28317157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82722024-02-06T22:19:11.073Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:10.283193727Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82732024-02-06T22:19:12.068Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.282915583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82742024-02-06T22:19:12.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.283017489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82752024-02-06T22:19:12.073Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.283673921Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82762024-02-06T22:19:12.073Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.283733933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82772024-02-06T22:19:12.073Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284205599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82782024-02-06T22:19:12.073Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284688224Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82792024-02-06T22:19:12.073Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284737077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82802024-02-06T22:19:12.073Z{"msg":"[0]R flush_numbers[0..12]: [143, 143, 143, 120, 136, 128, 136, 131, 138, 143, 135, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284764163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82812024-02-06T22:19:12.073Z{"msg":"[0]R generation[0..12]: [75, 75, 75, 64, 72, 68, 72, 70, 73, 75, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284799858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82822024-02-06T22:19:12.074Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284827655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82832024-02-06T22:19:12.074Z{"msg":"[1]R flush_numbers[0..12]: [143, 143, 143, 120, 136, 128, 136, 131, 138, 143, 135, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.28486364Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82842024-02-06T22:19:12.074Z{"msg":"[1]R generation[0..12]: [75, 75, 75, 64, 72, 68, 72, 70, 73, 75, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284889376Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82852024-02-06T22:19:12.074ZWait for a query_work_queue command to finish before sending IO
82862024-02-06T22:19:12.074Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.28493654Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82872024-02-06T22:19:12.074Z{"msg":"[2]R flush_numbers[0..12]: [143, 143, 143, 120, 136, 128, 136, 131, 138, 143, 135, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.284978674Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82882024-02-06T22:19:12.074Z{"msg":"[2]R generation[0..12]: [75, 75, 75, 64, 72, 68, 72, 70, 73, 75, 72, 75]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.28500895Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82892024-02-06T22:19:12.074Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285047325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82902024-02-06T22:19:12.074Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.28508516Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82912024-02-06T22:19:12.074Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285113056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82922024-02-06T22:19:12.074Z{"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285139183Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82932024-02-06T22:19:12.074Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285168099Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82942024-02-06T22:19:12.074Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285190016Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82952024-02-06T22:19:12.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285213962Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82962024-02-06T22:19:12.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.28523045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82972024-02-06T22:19:12.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285251688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82982024-02-06T22:19:12.074Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285265886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
82992024-02-06T22:19:12.074Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285278654Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
83002024-02-06T22:19:12.074Z{"msg":"a1b27ccd-a468-471a-aba4-bff07b460739 is now active with session: a932fef4-dbdb-468c-9e55-349d8db303b0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285298881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
83012024-02-06T22:19:12.074Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285311369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439,"session_id":"a932fef4-dbdb-468c-9e55-349d8db303b0"}
83022024-02-06T22:19:12.074Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:11.285323848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439}
83032024-02-06T22:19:12.074ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
83042024-02-06T22:19:12.074ZLoading write count information from file "/tmp/test_repair_verify.data"
83052024-02-06T22:19:12.074ZRun Repair workload
83062024-02-06T22:19:12.075Z01/30 Write block 167 len 40960 data: 4 5 7 6 4 7 9 8 7 7
83072024-02-06T22:19:12.078Z02/30 Read block 506 len 12288
83082024-02-06T22:19:12.078Z03/30 Write block 284 len 32768 data: 6 4 3 3 6 6 9 9
83092024-02-06T22:19:12.078Z04/30 Flush
83102024-02-06T22:19:12.078Z05/30 Write block 520 len 20480 data: 4 4 5 4 4
83112024-02-06T22:19:12.078Z06/30 Write block 257 len 28672 data: 4 3 4 5 7 7 6
83122024-02-06T22:19:12.082Z07/30 Read block 478 len 24576
83132024-02-06T22:19:12.082Z08/30 Write block 194 len 24576 data: 7 7 10 9 6 6
83142024-02-06T22:19:12.082Z09/30 Flush
83152024-02-06T22:19:12.085Z10/30 Write block 54 len 20480 data: 14 14 14 11 9
83162024-02-06T22:19:12.085Z11/30 Write block 389 len 4096 data: 5
83172024-02-06T22:19:12.085Z12/30 Read block 161 len 32768
83182024-02-06T22:19:12.125Z13/30 Read block 591 len 16384
83192024-02-06T22:19:12.125Z14/30 Write block 168 len 36864 data: 6 8 7 5 8 10 9 8 8
83202024-02-06T22:19:12.125Z15/30 Read block 378 len 28672
83212024-02-06T22:19:12.125Z16/30 Read block 46 len 32768
83222024-02-06T22:19:12.125Z17/30 Read block 563 len 8192
83232024-02-06T22:19:12.125Z18/30 Read block 296 len 36864
83242024-02-06T22:19:12.125Z19/30 Read block 348 len 8192
83252024-02-06T22:19:12.125Z20/30 Write block 234 len 4096 data: 9
83262024-02-06T22:19:12.125Z21/30 Read block 92 len 20480
83272024-02-06T22:19:12.125Z22/30 Write block 301 len 12288 data: 7 6 5
83282024-02-06T22:19:12.125Z23/30 Read block 6 len 20480
83292024-02-06T22:19:12.125Z24/30 Write block 503 len 24576 data: 9 7 7 8 8 7
83302024-02-06T22:19:12.125Z25/30 Read block 45 len 8192
83312024-02-06T22:19:12.125Z26/30 Write block 185 len 8192 data: 13 13
83322024-02-06T22:19:12.125Z27/30 Write block 348 len 20480 data: 11 10 13 15 14
83332024-02-06T22:19:12.126Z28/30 Read block 377 len 36864
83342024-02-06T22:19:12.126Z29/30 Write block 108 len 4096 data: 9
83352024-02-06T22:19:12.126Z30/30 Write block 407 len 36864 data: 5 5 7 8 9 6 5 6 8
83362024-02-06T22:19:12.126Z----------------------------------------------------------------
83372024-02-06T22:19:12.126Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
83382024-02-06T22:19:12.126ZStates: Active Active Active
83392024-02-06T22:19:12.126ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
83402024-02-06T22:19:12.126Z 1 Acked 1000 Write 10 Done Sent Done false
83412024-02-06T22:19:12.126Z 2 Acked 1001 Read 3 Done Sent Done false
83422024-02-06T22:19:12.126Z 3 Acked 1002 Write 8 Done Sent Done false
83432024-02-06T22:19:12.126Z 4 Acked 1003 Flush 0 Done Sent Done false
83442024-02-06T22:19:12.126Z 5 Acked 1004 Write 5 Done Sent Done false
83452024-02-06T22:19:12.126Z 6 Acked 1005 Write 7 Done Sent Done false
83462024-02-06T22:19:12.126Z 7 Acked 1006 Read 6 Done Sent Done false
83472024-02-06T22:19:12.126Z 8 Acked 1007 Write 6 Done Sent Done false
83482024-02-06T22:19:12.126Z 9 Acked 1008 Flush 0 Done Sent Done false
83492024-02-06T22:19:12.126Z 10 Acked 1009 Write 5 Done Sent Done false
83502024-02-06T22:19:12.126Z 11 Acked 1010 Write 1 Done Sent Done false
83512024-02-06T22:19:12.126Z 12 Acked 1011 Read 8 Done Sent Done false
83522024-02-06T22:19:12.126Z 13 Acked 1012 Read 4 Done Sent Done false
83532024-02-06T22:19:12.126Z 14 Acked 1013 Write 9 Done Sent Done false
83542024-02-06T22:19:12.126Z 15 Acked 1014 Read 7 Done Sent Done false
83552024-02-06T22:19:12.126Z 16 Acked 1015 Read 8 Done Sent Done false
83562024-02-06T22:19:12.126Z 17 Acked 1016 Read 2 Done Sent Done false
83572024-02-06T22:19:12.126Z 18 Acked 1017 Read 9 Done Sent Done false
83582024-02-06T22:19:12.127Z 19 Acked 1018 Read 2 Done Sent Done false
83592024-02-06T22:19:12.127Z 20 Acked 1019 Write 1 Done Sent Done false
83602024-02-06T22:19:12.127Z 21 Acked 1020 Read 5 Done Sent Done false
83612024-02-06T22:19:12.127Z 22 Acked 1021 Write 3 Done Sent Done false
83622024-02-06T22:19:12.127Z 23 Acked 1022 Read 5 Done Sent Done false
83632024-02-06T22:19:12.127Z 24 Acked 1023 Write 6 Done Sent Done false
83642024-02-06T22:19:12.127Z 25 Acked 1024 Read 2 Done Sent Done false
83652024-02-06T22:19:12.127Z 26 Acked 1025 Write 2 Done Sent Done false
83662024-02-06T22:19:12.127Z 27 Acked 1026 Write 5 Done Sent Done false
83672024-02-06T22:19:12.127Z 28 Acked 1027 Read 9 Done Sent Done false
83682024-02-06T22:19:12.127Z 29 Acked 1028 Write 1 Done Sent Done false
83692024-02-06T22:19:12.127Z 30 Acked 1029 Write 9 Sent Sent Sent false
83702024-02-06T22:19:12.127Z STATES DS:0 DS:1 DS:2 TOTAL
83712024-02-06T22:19:12.127Z New 0 0 0 0
83722024-02-06T22:19:12.127Z Sent 1 30 1 32
83732024-02-06T22:19:12.127Z Done 29 0 29 58
83742024-02-06T22:19:12.127Z Skipped 0 0 0 0
83752024-02-06T22:19:12.127Z Error 0 0 0 0
83762024-02-06T22:19:12.127ZLast Flush: 1008 0 1008
83772024-02-06T22:19:12.127ZDownstairs last five completed:
83782024-02-06T22:19:12.127ZUpstairs last five completed: 30 29 28 27 26
83792024-02-06T22:19:12.127Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:11.314558388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1439}
83802024-02-06T22:19:12.127ZWrote out file "/tmp/test_repair_verify.data"
83812024-02-06T22:19:12.127Z
83822024-02-06T22:19:12.127ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83832024-02-06T22:19:12.132ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83842024-02-06T22:19:12.132Z 0 000-019 75 75 75 143 143 143 F F F
83852024-02-06T22:19:12.132Z 1 020-039 75 75 75 143 143 143 F F F
83862024-02-06T22:19:12.132Z 2 040-059 75 75 75 143 143 143 T F T <---
83872024-02-06T22:19:12.132Z 3 060-079 64 64 64 120 120 120 F F F
83882024-02-06T22:19:12.133Z 4 080-099 72 72 72 136 136 136 F F F
83892024-02-06T22:19:12.133Z 5 100-119 68 68 68 128 128 128 T F T <---
83902024-02-06T22:19:12.133Z 6 120-139 72 72 72 136 136 136 F F F
83912024-02-06T22:19:12.133Z 7 140-159 70 70 70 131 131 131 F F F
83922024-02-06T22:19:12.133Z 8 160-179 76 73 76 144 138 144 T F T <---
83932024-02-06T22:19:12.133Z 9 180-199 76 75 76 145 143 145 T F T <---
83942024-02-06T22:19:12.133Z 10 200-219 72 72 72 135 135 135 F F F
83952024-02-06T22:19:12.133Z 11 220-239 75 75 75 143 143 143 T F T <---
83962024-02-06T22:19:12.133Z 12 240-259 76 64 76 145 120 145 F F F <---
83972024-02-06T22:19:12.133Z 13 260-279 76 74 76 145 142 145 F F F <---
83982024-02-06T22:19:12.133Z 14 280-299 76 75 76 144 143 144 F F F <---
83992024-02-06T22:19:12.133Z 15 300-319 75 75 75 143 143 143 T F T <---
84002024-02-06T22:19:12.133Z 16 320-339 62 62 62 117 117 117 F F F
84012024-02-06T22:19:12.133Z 17 340-359 75 75 75 143 143 143 T F T <---
84022024-02-06T22:19:12.133Z 18 360-379 67 67 67 126 126 126 F F F
84032024-02-06T22:19:12.133Z 19 380-399 75 75 75 143 143 143 T F T <---
84042024-02-06T22:19:12.133Z 20 400-419 75 75 75 143 143 143 T F T <---
84052024-02-06T22:19:12.133Z 21 420-439 73 73 73 138 138 138 F F F
84062024-02-06T22:19:12.133Z 22 440-459 58 58 58 113 113 113 F F F
84072024-02-06T22:19:12.133Z 23 460-479 58 58 58 113 113 113 F F F
84082024-02-06T22:19:12.133Z 24 480-499 72 72 72 136 136 136 F F F
84092024-02-06T22:19:12.133Z 25 500-519 70 70 70 131 131 131 T F T <---
84102024-02-06T22:19:12.133Z 26 520-539 76 74 76 145 141 145 F F F <---
84112024-02-06T22:19:12.134Z 27 540-559 68 68 68 128 128 128 F F F
84122024-02-06T22:19:12.134Z 28 560-579 75 75 75 143 143 143 F F F
84132024-02-06T22:19:12.134Z 29 580-599 66 66 66 121 121 121 F F F
84142024-02-06T22:19:12.134ZMax gen: 76, Max flush: 145
84152024-02-06T22:19:12.134ZError: Difference in extent metadata found!
84162024-02-06T22:19:12.134ZA Difference in extent metadata is expected here
84172024-02-06T22:19:12.134ZOn loop 38
84182024-02-06T22:19:12.134Z
84192024-02-06T22:19:12.137ZVerifying data now
84202024-02-06T22:19:16.823ZNo change after verify
84212024-02-06T22:19:16.823ZLoop: 38 Downstairs dump after verify (and repair):
84222024-02-06T22:19:16.837ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84232024-02-06T22:19:16.837Z 0 000-019 75 75 75 143 143 143 F F F
84242024-02-06T22:19:16.837Z 1 020-039 75 75 75 143 143 143 F F F
84252024-02-06T22:19:16.837Z 2 040-059 77 77 77 146 146 146 F F F
84262024-02-06T22:19:16.837Z 3 060-079 64 64 64 120 120 120 F F F
84272024-02-06T22:19:16.837Z 4 080-099 72 72 72 136 136 136 F F F
84282024-02-06T22:19:16.838Z 5 100-119 77 77 77 146 146 146 F F F
84292024-02-06T22:19:16.845Z 6 120-139 72 72 72 136 136 136 F F F
84302024-02-06T22:19:16.845Z 7 140-159 70 70 70 131 131 131 F F F
84312024-02-06T22:19:16.845Z 8 160-179 77 77 77 146 146 146 F F F
84322024-02-06T22:19:16.845Z 9 180-199 77 77 77 146 146 146 F F F
84332024-02-06T22:19:16.845Z 10 200-219 72 72 72 135 135 135 F F F
84342024-02-06T22:19:16.845Z 11 220-239 77 77 77 146 146 146 F F F
84352024-02-06T22:19:16.845Z 12 240-259 76 76 76 145 145 145 F F F
84362024-02-06T22:19:16.845Z 13 260-279 76 76 76 145 145 145 F F F
84372024-02-06T22:19:16.845Z 14 280-299 76 76 76 144 144 144 F F F
84382024-02-06T22:19:16.845Z 15 300-319 77 77 77 146 146 146 F F F
84392024-02-06T22:19:16.845Z 16 320-339 62 62 62 117 117 117 F F F
84402024-02-06T22:19:16.846Z 17 340-359 77 77 77 146 146 146 F F F
84412024-02-06T22:19:16.846Z 18 360-379 67 67 67 126 126 126 F F F
84422024-02-06T22:19:16.846Z 19 380-399 77 77 77 146 146 146 F F F
84432024-02-06T22:19:16.846Z 20 400-419 77 77 77 146 146 146 F F F
84442024-02-06T22:19:16.846Z 21 420-439 73 73 73 138 138 138 F F F
84452024-02-06T22:19:16.846Z 22 440-459 58 58 58 113 113 113 F F F
84462024-02-06T22:19:16.846Z 23 460-479 58 58 58 113 113 113 F F F
84472024-02-06T22:19:16.846Z 24 480-499 72 72 72 136 136 136 F F F
84482024-02-06T22:19:16.846Z 25 500-519 77 77 77 146 146 146 F F F
84492024-02-06T22:19:16.846Z 26 520-539 76 76 76 145 145 145 F F F
84502024-02-06T22:19:16.846Z 27 540-559 68 68 68 128 128 128 F F F
84512024-02-06T22:19:16.846Z 28 560-579 75 75 75 143 143 143 F F F
84522024-02-06T22:19:16.846Z 29 580-599 66 66 66 121 121 121 F F F
84532024-02-06T22:19:16.846ZMax gen: 77, Max flush: 146
84542024-02-06T22:19:16.847Z
84552024-02-06T22:19:16.847ZBegin loop 39
84562024-02-06T22:19:16.847ZDownstairs to restart: 1
84572024-02-06T22:19:16.858Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069307232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447}
84582024-02-06T22:19:16.870Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069535292Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447}
84592024-02-06T22:19:16.870Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069574146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447}
84602024-02-06T22:19:16.870Z{"msg":"Crucible fd80df3f-95ae-4e5d-9f68-91d2edf17aa4 has session id: 5de0d0d2-3601-4d59-a3fc-351a8a9f05d9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069600063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447Crucible runtime is spawned
84612024-02-06T22:19:16.871Z,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84622024-02-06T22:19:16.871Z{"msg":"Upstairs opts: Upstairs UUID: fd80df3f-95ae-4e5d-9f68-91d2edf17aa4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069640557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84632024-02-06T22:19:16.871Z{"msg":"Crucible stats registered with UUID: fd80df3f-95ae-4e5d-9f68-91d2edf17aa4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069665804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84642024-02-06T22:19:16.871Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.06969709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447}
84652024-02-06T22:19:16.871Z{"msg":"fd80df3f-95ae-4e5d-9f68-91d2edf17aa4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069876106Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84662024-02-06T22:19:16.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069908191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84672024-02-06T22:19:16.872Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069943756Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84682024-02-06T22:19:16.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.069981581Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84692024-02-06T22:19:16.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070039644Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84702024-02-06T22:19:16.873Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070084517Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84712024-02-06T22:19:16.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070130081Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84722024-02-06T22:19:16.873Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070155658Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84732024-02-06T22:19:16.873Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070200202Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84742024-02-06T22:19:16.874Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070227938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84752024-02-06T22:19:16.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070251965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84762024-02-06T22:19:16.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070275921Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84772024-02-06T22:19:16.874Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:16.070304198Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84782024-02-06T22:19:16.874Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:16.070333104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84792024-02-06T22:19:16.874Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:16.070460876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84802024-02-06T22:19:16.875Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.07051027Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84812024-02-06T22:19:16.875Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070542996Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84822024-02-06T22:19:16.875Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070568072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84832024-02-06T22:19:16.875Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070909666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84842024-02-06T22:19:16.875Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.070967548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84852024-02-06T22:19:16.875Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.071060595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84862024-02-06T22:19:16.876Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.071102Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84872024-02-06T22:19:16.876Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.071475639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84882024-02-06T22:19:16.876Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.071646886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84892024-02-06T22:19:16.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.071966963Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84902024-02-06T22:19:16.876Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.072005548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84912024-02-06T22:19:16.876Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.072023236Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84922024-02-06T22:19:16.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.072174205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84932024-02-06T22:19:16.876Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.072223478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84942024-02-06T22:19:16.877Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:16.072259304Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84952024-02-06T22:19:17.859Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.07204281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84962024-02-06T22:19:17.859Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.07219238Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84972024-02-06T22:19:17.864Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.073068512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84982024-02-06T22:19:17.864Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.073115905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
84992024-02-06T22:19:17.864Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.073558956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85002024-02-06T22:19:17.864Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074164694Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85012024-02-06T22:19:17.864Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074210468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85022024-02-06T22:19:17.864Z{"msg":"[0]R flush_numbers[0..12]: [143, 143, 146, 120, 136, 146, 136, 131, 146, 146, 135, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074228155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85032024-02-06T22:19:17.865Z{"msg":"[0]R generation[0..12]: [75, 75, 77, 64, 72, 77, 72, 70, 77, 77, 72, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074250632Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85042024-02-06T22:19:17.865Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.07426596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85052024-02-06T22:19:17.865Z{"msg":"[1]R flush_numbers[0..12]: [143, 143, 146, 120, 136, 146, 136, 131, 146, 146, 135, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074280998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85062024-02-06T22:19:17.865ZWait for a query_work_queue command to finish before sending IO
85072024-02-06T22:19:17.865Z{"msg":"[1]R generation[0..12]: [75, 75, 77, 64, 72, 77, 72, 70, 77, 77, 72, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074312984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85082024-02-06T22:19:17.865Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074327992Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85092024-02-06T22:19:17.865Z{"msg":"[2]R flush_numbers[0..12]: [143, 143, 146, 120, 136, 146, 136, 131, 146, 146, 135, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074350949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85102024-02-06T22:19:17.865Z{"msg":"[2]R generation[0..12]: [75, 75, 77, 64, 72, 77, 72, 70, 77, 77, 72, 77]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074365247Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85112024-02-06T22:19:17.865Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074379455Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85122024-02-06T22:19:17.865Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074393073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85132024-02-06T22:19:17.865Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.07441493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85142024-02-06T22:19:17.865Z{"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074428588Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85152024-02-06T22:19:17.865Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074441966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85162024-02-06T22:19:17.865Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074463214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85172024-02-06T22:19:17.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074475902Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85182024-02-06T22:19:17.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.0744899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85192024-02-06T22:19:17.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074503708Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85202024-02-06T22:19:17.865Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074517036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85212024-02-06T22:19:17.865Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074536524Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85222024-02-06T22:19:17.865Z{"msg":"fd80df3f-95ae-4e5d-9f68-91d2edf17aa4 is now active with session: 5de0d0d2-3601-4d59-a3fc-351a8a9f05d9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074550012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85232024-02-06T22:19:17.865Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.07456224Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447,"session_id":"5de0d0d2-3601-4d59-a3fc-351a8a9f05d9"}
85242024-02-06T22:19:17.865Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:17.074575638Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447}
85252024-02-06T22:19:17.865ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
85262024-02-06T22:19:17.866ZLoading write count information from file "/tmp/test_repair_verify.data"
85272024-02-06T22:19:17.866ZRun Repair workload
85282024-02-06T22:19:17.866Z01/30 Write block 539 len 24576 data: 3 3 5 5 6 7
85292024-02-06T22:19:17.869Z02/30 Read block 144 len 24576
85302024-02-06T22:19:17.869Z03/30 Write block 70 len 32768 data: 6 4 5 5 7 6 3 4
85312024-02-06T22:19:17.869Z04/30 Flush
85322024-02-06T22:19:17.870Z05/30 Write block 157 len 16384 data: 9 7 7 6
85332024-02-06T22:19:17.870Z06/30 Read block 312 len 24576
85342024-02-06T22:19:17.873Z07/30 Read block 369 len 24576
85352024-02-06T22:19:17.873Z08/30 Write block 510 len 36864 data: 6 6 5 8 8 7 7 7 6
85362024-02-06T22:19:17.873Z09/30 Write block 381 len 12288 data: 9 8 8
85372024-02-06T22:19:17.873Z10/30 Write block 551 len 32768 data: 5 5 5 5 4 5 5 6
85382024-02-06T22:19:17.877Z11/30 Read block 475 len 20480
85392024-02-06T22:19:17.877Z12/30 Write block 52 len 12288 data: 11 13 15
85402024-02-06T22:19:17.877Z13/30 Write block 124 len 36864 data: 9 10 10 10 11 8 9 8 6
85412024-02-06T22:19:17.878Z14/30 Read block 414 len 4096
85422024-02-06T22:19:17.881Z15/30 Read block 308 len 12288
85432024-02-06T22:19:17.881Z16/30 Write block 574 len 24576 data: 10 9 10 11 10 9
85442024-02-06T22:19:17.881Z17/30 Write block 162 len 4096 data: 6
85452024-02-06T22:19:17.881Z18/30 Write block 530 len 8192 data: 10 10
85462024-02-06T22:19:17.881Z19/30 Read block 355 len 8192
85472024-02-06T22:19:17.881Z20/30 Read block 531 len 40960
85482024-02-06T22:19:17.886Z21/30 Write block 127 len 32768 data: 11 12 9 10 9 7 7 6
85492024-02-06T22:19:17.886Z22/30 Read block 70 len 28672
85502024-02-06T22:19:17.886Z23/30 Write block 228 len 32768 data: 6 5 5 8 8 8 10 9
85512024-02-06T22:19:17.886Z24/30 Read block 528 len 4096
85522024-02-06T22:19:17.889Z25/30 Write block 429 len 40960 data: 7 7 3 4 5 6 7 9 9 8
85532024-02-06T22:19:17.889Z26/30 Write block 261 len 32768 data: 8 8 7 5 6 8 8 8
85542024-02-06T22:19:17.889Z27/30 Read block 36 len 8192
85552024-02-06T22:19:17.889Z28/30 Write block 467 len 36864 data: 6 8 7 7 8 8 6 6 6
85562024-02-06T22:19:17.889Z29/30 Read block 236 len 28672
85572024-02-06T22:19:17.892Z30/30 Write block 89 len 24576 data: 5 5 6 3 3 4
85582024-02-06T22:19:17.892Z----------------------------------------------------------------
85592024-02-06T22:19:17.892Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:26
85602024-02-06T22:19:17.892ZStates: Active Active Active
85612024-02-06T22:19:17.892ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85622024-02-06T22:19:17.892Z 5 Acked 1004 Write 4 Done Done Done false
85632024-02-06T22:19:17.893Z 6 Acked 1005 Read 6 Done Done Done false
85642024-02-06T22:19:17.893Z 7 Acked 1006 Read 6 Done Done Done false
85652024-02-06T22:19:17.893Z 8 Acked 1007 Write 9 Done Done Done false
85662024-02-06T22:19:17.893Z 9 Acked 1008 Write 3 Done Done Done false
85672024-02-06T22:19:17.893Z 10 Acked 1009 Write 8 Done Done Done false
85682024-02-06T22:19:17.893Z 11 Acked 1010 Read 5 Done Done Done false
85692024-02-06T22:19:17.893Z 12 Acked 1011 Write 3 Done Done Done false
85702024-02-06T22:19:17.893Z 13 Acked 1012 Write 9 Done Done Done false
85712024-02-06T22:19:17.893Z 14 Acked 1013 Read 1 Done Sent Done false
85722024-02-06T22:19:17.893Z 15 Acked 1014 Read 3 Done Sent Done false
85732024-02-06T22:19:17.893Z 16 Acked 1015 Write 6 Done Sent Done false
85742024-02-06T22:19:17.893Z 17 Acked 1016 Write 1 Done Sent Done false
85752024-02-06T22:19:17.893Z 18 Acked 1017 Write 2 Done Sent Done false
85762024-02-06T22:19:17.893Z 19 Acked 1018 Read 2 Done Sent Done false
85772024-02-06T22:19:17.893Z 20 Acked 1019 Read 10 Done Sent Done false
85782024-02-06T22:19:17.893Z 21 Acked 1020 Write 8 Done Sent Done false
85792024-02-06T22:19:17.893Z 22 Acked 1021 Read 7 Done Sent Done false
85802024-02-06T22:19:17.893Z 23 Acked 1022 Write 8 Done Sent Done false
85812024-02-06T22:19:17.893Z 24 Acked 1023 Read 1 Done Sent Done false
85822024-02-06T22:19:17.893Z 25 Acked 1024 Write 10 Done Sent Done false
85832024-02-06T22:19:17.893Z 26 Acked 1025 Write 8 Done Sent Done false
85842024-02-06T22:19:17.893Z 27 Acked 1026 Read 2 Done Sent Done false
85852024-02-06T22:19:17.893Z 28 Acked 1027 Write 9 Done Sent Done false
85862024-02-06T22:19:17.894Z 29 Acked 1028 Read 7 Done Sent Done false
85872024-02-06T22:19:17.894Z 30 Acked 1029 Write 6 Sent Sent Sent false
85882024-02-06T22:19:17.894Z STATES DS:0 DS:1 DS:2 TOTAL
85892024-02-06T22:19:17.894Z New 0 0 0 0
85902024-02-06T22:19:17.894Z Sent 1 17 1 19
85912024-02-06T22:19:17.894Z Done 25 9 25 59
85922024-02-06T22:19:17.894Z Skipped 0 0 0 0
85932024-02-06T22:19:17.894Z Error 0 0 0 0
85942024-02-06T22:19:17.894ZLast Flush: 1003 1003 1003
85952024-02-06T22:19:17.894ZDownstairs last five completed: 1003 1002 1001 1000
85962024-02-06T22:19:17.894ZUpstairs last five completed: 30 29 28 27 26
85972024-02-06T22:19:17.894Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:17.105042152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1447}
85982024-02-06T22:19:17.894ZWrote out file "/tmp/test_repair_verify.data"
85992024-02-06T22:19:17.902Z
86002024-02-06T22:19:17.905ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
86012024-02-06T22:19:17.921ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86022024-02-06T22:19:17.921Z 0 000-019 75 75 75 143 143 143 F F F
86032024-02-06T22:19:17.921Z 1 020-039 75 75 75 143 143 143 F F F
86042024-02-06T22:19:17.921Z 2 040-059 77 77 77 146 146 146 T T T
86052024-02-06T22:19:17.921Z 3 060-079 78 78 78 147 147 147 F F F
86062024-02-06T22:19:17.921Z 4 080-099 72 72 72 136 136 136 T F T <---
86072024-02-06T22:19:17.921Z 5 100-119 77 77 77 146 146 146 F F F
86082024-02-06T22:19:17.921Z 6 120-139 72 72 72 136 136 136 T T T
86092024-02-06T22:19:17.921Z 7 140-159 70 70 70 131 131 131 T T T
86102024-02-06T22:19:17.921Z 8 160-179 77 77 77 146 146 146 T T T
86112024-02-06T22:19:17.921Z 9 180-199 77 77 77 146 146 146 F F F
86122024-02-06T22:19:17.921Z 10 200-219 72 72 72 135 135 135 F F F
86132024-02-06T22:19:17.921Z 11 220-239 77 77 77 146 146 146 T F T <---
86142024-02-06T22:19:17.921Z 12 240-259 76 76 76 145 145 145 F F F
86152024-02-06T22:19:17.921Z 13 260-279 76 76 76 145 145 145 T F T <---
86162024-02-06T22:19:17.921Z 14 280-299 76 76 76 144 144 144 F F F
86172024-02-06T22:19:17.921Z 15 300-319 77 77 77 146 146 146 F F F
86182024-02-06T22:19:17.921Z 16 320-339 62 62 62 117 117 117 F F F
86192024-02-06T22:19:17.921Z 17 340-359 77 77 77 146 146 146 F F F
86202024-02-06T22:19:17.925Z 18 360-379 67 67 67 126 126 126 F F F
86212024-02-06T22:19:17.925Z 19 380-399 77 77 77 146 146 146 T T T
86222024-02-06T22:19:17.925Z 20 400-419 77 77 77 146 146 146 F F F
86232024-02-06T22:19:17.925Z 21 420-439 73 73 73 138 138 138 T F T <---
86242024-02-06T22:19:17.925Z 22 440-459 58 58 58 113 113 113 F F F
86252024-02-06T22:19:17.925Z 23 460-479 58 58 58 113 113 113 T F T <---
86262024-02-06T22:19:17.925Z 24 480-499 72 72 72 136 136 136 F F F
86272024-02-06T22:19:17.925Z 25 500-519 77 77 77 146 146 146 T T T
86282024-02-06T22:19:17.925Z 26 520-539 78 78 78 147 147 147 T F T <---
86292024-02-06T22:19:17.925Z 27 540-559 78 78 78 147 147 147 T T T
86302024-02-06T22:19:17.926Z 28 560-579 75 75 75 143 143 143 T F T <---
86312024-02-06T22:19:17.926Z 29 580-599 66 66 66 121 121 121 F F F
86322024-02-06T22:19:17.926ZMax gen: 78, Max flush: 147
86332024-02-06T22:19:17.926ZError: Difference in extent metadata found!
86342024-02-06T22:19:17.926ZA Difference in extent metadata is expected here
86352024-02-06T22:19:17.926ZOn loop 39
86362024-02-06T22:19:17.926Z
86372024-02-06T22:19:17.929ZVerifying data now
86382024-02-06T22:19:23.634ZNo change after verify
86392024-02-06T22:19:23.638ZLoop: 39 Downstairs dump after verify (and repair):
86402024-02-06T22:19:23.650ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86412024-02-06T22:19:23.650Z 0 000-019 75 75 75 143 143 143 F F F
86422024-02-06T22:19:23.650Z 1 020-039 75 75 75 143 143 143 F F F
86432024-02-06T22:19:23.650Z 2 040-059 79 79 79 148 148 148 F F F
86442024-02-06T22:19:23.650Z 3 060-079 78 78 78 147 147 147 F F F
86452024-02-06T22:19:23.650Z 4 080-099 79 79 79 148 148 148 F F F
86462024-02-06T22:19:23.650Z 5 100-119 77 77 77 146 146 146 F F F
86472024-02-06T22:19:23.650Z 6 120-139 79 79 79 148 148 148 F F F
86482024-02-06T22:19:23.650Z 7 140-159 79 79 79 148 148 148 F F F
86492024-02-06T22:19:23.650Z 8 160-179 79 79 79 148 148 148 F F F
86502024-02-06T22:19:23.650Z 9 180-199 77 77 77 146 146 146 F F F
86512024-02-06T22:19:23.654Z 10 200-219 72 72 72 135 135 135 F F F
86522024-02-06T22:19:23.654Z 11 220-239 79 79 79 148 148 148 F F F
86532024-02-06T22:19:23.654Z 12 240-259 76 76 76 145 145 145 F F F
86542024-02-06T22:19:23.654Z 13 260-279 79 79 79 148 148 148 F F F
86552024-02-06T22:19:23.654Z 14 280-299 76 76 76 144 144 144 F F F
86562024-02-06T22:19:23.654Z 15 300-319 77 77 77 146 146 146 F F F
86572024-02-06T22:19:23.654Z 16 320-339 62 62 62 117 117 117 F F F
86582024-02-06T22:19:23.654Z 17 340-359 77 77 77 146 146 146 F F F
86592024-02-06T22:19:23.654Z 18 360-379 67 67 67 126 126 126 F F F
86602024-02-06T22:19:23.654Z 19 380-399 79 79 79 148 148 148 F F F
86612024-02-06T22:19:23.654Z 20 400-419 77 77 77 146 146 146 F F F
86622024-02-06T22:19:23.654Z 21 420-439 79 79 79 148 148 148 F F F
86632024-02-06T22:19:23.655Z 22 440-459 58 58 58 113 113 113 F F F
86642024-02-06T22:19:23.655Z 23 460-479 79 79 79 148 148 148 F F F
86652024-02-06T22:19:23.655Z 24 480-499 72 72 72 136 136 136 F F F
86662024-02-06T22:19:23.655Z 25 500-519 79 79 79 148 148 148 F F F
86672024-02-06T22:19:23.655Z 26 520-539 79 79 79 148 148 148 F F F
86682024-02-06T22:19:23.655Z 27 540-559 79 79 79 148 148 148 F F F
86692024-02-06T22:19:23.655Z 28 560-579 79 79 79 148 148 148 F F F
86702024-02-06T22:19:23.655Z 29 580-599 66 66 66 121 121 121 F F F
86712024-02-06T22:19:23.655ZMax gen: 79, Max flush: 148
86722024-02-06T22:19:23.655Z
86732024-02-06T22:19:23.655ZBegin loop 40
86742024-02-06T22:19:23.655ZDownstairs to restart: 0
86752024-02-06T22:19:23.671Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884287145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455}
86762024-02-06T22:19:23.678Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884558269Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455}
86772024-02-06T22:19:23.678Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
86782024-02-06T22:19:23.678Z,"time":"2024-02-06T22:19:22.884592914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455}
86792024-02-06T22:19:23.678Z{"msg":"Crucible ef9a3d19-f0dc-4480-b1ba-6005bc8db2da has session id: 51a4f7b6-8d29-4a96-9130-a75467f385e7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884629899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86802024-02-06T22:19:23.678Z{"msg":"Upstairs opts: Upstairs UUID: ef9a3d19-f0dc-4480-b1ba-6005bc8db2da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884675143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86812024-02-06T22:19:23.678Z{"msg":"Crucible stats registered with UUID: ef9a3d19-f0dc-4480-b1ba-6005bc8db2da","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884713568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86822024-02-06T22:19:23.678Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884756662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455}
86832024-02-06T22:19:23.678Z{"msg":"ef9a3d19-f0dc-4480-b1ba-6005bc8db2da active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884791817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86842024-02-06T22:19:23.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884820433Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86852024-02-06T22:19:23.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884840211Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86862024-02-06T22:19:23.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884865018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86872024-02-06T22:19:23.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884898093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86882024-02-06T22:19:23.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.884954645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86892024-02-06T22:19:23.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885128552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86902024-02-06T22:19:23.679Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885190234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86912024-02-06T22:19:23.679Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885252505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86922024-02-06T22:19:23.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.8852898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86932024-02-06T22:19:23.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885327295Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86942024-02-06T22:19:23.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.88536132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86952024-02-06T22:19:23.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885406594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86962024-02-06T22:19:23.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885724811Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86972024-02-06T22:19:23.679Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885753867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86982024-02-06T22:19:23.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885769475Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
86992024-02-06T22:19:23.679Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.885783213Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87002024-02-06T22:19:23.679Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.886336499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87012024-02-06T22:19:23.679Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.886498577Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87022024-02-06T22:19:23.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.886954825Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87032024-02-06T22:19:23.679Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.887017836Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87042024-02-06T22:19:23.679Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.887056851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87052024-02-06T22:19:23.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.887073039Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87062024-02-06T22:19:23.679Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.887087457Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87072024-02-06T22:19:23.679Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.887109074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87082024-02-06T22:19:23.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.887122402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87092024-02-06T22:19:23.679Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.88714411Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87102024-02-06T22:19:23.685Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.887601388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87112024-02-06T22:19:23.685Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.88802606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87122024-02-06T22:19:23.685Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888071674Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87132024-02-06T22:19:23.685Z{"msg":"[0]R flush_numbers[0..12]: [143, 143, 148, 147, 148, 146, 148, 148, 148, 146, 135, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888094241Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87142024-02-06T22:19:23.685Z{"msg":"[0]R generation[0..12]: [75, 75, 79, 78, 79, 77, 79, 79, 79, 77, 72, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888118668Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87152024-02-06T22:19:23.685Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888134546Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87162024-02-06T22:19:23.685Z{"msg":"[1]R flush_numbers[0..12]: [143, 143, 148, 147, 148, 146, 148, 148, 148, 146, 135, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888157353Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87172024-02-06T22:19:23.685Z{"msg":"[1]R generation[0..12]: [75, 75, 79, 78, 79, 77, 79, 79, 79, 77, 72, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888171601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87182024-02-06T22:19:23.685ZWait for a query_work_queue command to finish before sending IO
87192024-02-06T22:19:23.685Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888241531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87202024-02-06T22:19:23.685Z{"msg":"[2]R flush_numbers[0..12]: [143, 143, 148, 147, 148, 146, 148, 148, 148, 146, 135, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888278536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87212024-02-06T22:19:23.685Z{"msg":"[2]R generation[0..12]: [75, 75, 79, 78, 79, 77, 79, 79, 79, 77, 72, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888306972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87222024-02-06T22:19:23.685Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888330619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87232024-02-06T22:19:23.685Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888344917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87242024-02-06T22:19:23.685Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888358775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87252024-02-06T22:19:23.685Z{"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888379563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87262024-02-06T22:19:23.686Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888394151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87272024-02-06T22:19:23.686Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888407849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87282024-02-06T22:19:23.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888420747Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87292024-02-06T22:19:23.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888437745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87302024-02-06T22:19:23.686Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888461012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87312024-02-06T22:19:23.686Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888490788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87322024-02-06T22:19:23.686Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888513425Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87332024-02-06T22:19:23.686Z{"msg":"ef9a3d19-f0dc-4480-b1ba-6005bc8db2da is now active with session: 51a4f7b6-8d29-4a96-9130-a75467f385e7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888542681Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87342024-02-06T22:19:23.686Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888564587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455,"session_id":"51a4f7b6-8d29-4a96-9130-a75467f385e7"}
87352024-02-06T22:19:23.686Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:22.888591424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455}
87362024-02-06T22:19:23.686ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
87372024-02-06T22:19:23.686ZLoading write count information from file "/tmp/test_repair_verify.data"
87382024-02-06T22:19:23.686ZRun Repair workload
87392024-02-06T22:19:23.686Z01/30 Write block 109 len 28672 data: 6 6 4 6 8 10 9
87402024-02-06T22:19:23.686Z02/30 Flush
87412024-02-06T22:19:23.686Z03/30 Write block 398 len 40960 data: 6 7 8 9 9 7 6 5 6 6
87422024-02-06T22:19:23.686Z04/30 Read block 28 len 28672
87432024-02-06T22:19:23.686Z05/30 Read block 130 len 24576
87442024-02-06T22:19:23.686Z06/30 Read block 10 len 36864
87452024-02-06T22:19:23.690Z07/30 Read block 290 len 28672
87462024-02-06T22:19:23.690Z08/30 Read block 111 len 28672
87472024-02-06T22:19:23.690Z09/30 Read block 167 len 20480
87482024-02-06T22:19:23.690Z10/30 Read block 93 len 28672
87492024-02-06T22:19:23.690Z11/30 Read block 254 len 40960
87502024-02-06T22:19:23.690Z12/30 Write block 82 len 8192 data: 6 6
87512024-02-06T22:19:23.690Z13/30 Write block 507 len 4096 data: 9
87522024-02-06T22:19:23.690Z14/30 Flush
87532024-02-06T22:19:23.693Z15/30 Write block 457 len 12288 data: 3 3 3
87542024-02-06T22:19:23.693Z16/30 Read block 524 len 4096
87552024-02-06T22:19:23.693Z17/30 Read block 490 len 4096
87562024-02-06T22:19:23.696Z18/30 Write block 271 len 32768 data: 5 6 4 5 7 8 8 9
87572024-02-06T22:19:23.696Z19/30 Write block 167 len 16384 data: 5 7 9 8
87582024-02-06T22:19:23.696Z20/30 Read block 394 len 24576
87592024-02-06T22:19:23.697Z21/30 Read block 490 len 24576
87602024-02-06T22:19:23.700Z22/30 Read block 331 len 40960
87612024-02-06T22:19:23.700Z23/30 Write block 439 len 24576 data: 7 5 3 5 5 4
87622024-02-06T22:19:23.700Z24/30 Write block 585 len 40960 data: 6 4 7 7 9 11 10 9 11 10
87632024-02-06T22:19:23.703Z25/30 Read block 402 len 20480
87642024-02-06T22:19:23.703Z26/30 Write block 391 len 28672 data: 6 5 5 4 5 4 5
87652024-02-06T22:19:23.703Z27/30 Write block 472 len 32768 data: 9 7 7 7 7 8 8 6
87662024-02-06T22:19:23.703Z28/30 Write block 227 len 24576 data: 5 7 6 6 9 9
87672024-02-06T22:19:23.707Z29/30 Write block 288 len 32768 data: 7 7 10 10 9 10 10 11
87682024-02-06T22:19:23.707Z30/30 Read block 58 len 28672
87692024-02-06T22:19:23.707Z----------------------------------------------------------------
87702024-02-06T22:19:23.707Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
87712024-02-06T22:19:23.707ZStates: Active Active Active
87722024-02-06T22:19:23.707ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87732024-02-06T22:19:23.707Z 1 Acked 1000 Write 7 Sent Done Done false
87742024-02-06T22:19:23.707Z 2 Acked 1001 Flush 0 Sent Done Done false
87752024-02-06T22:19:23.707Z 3 Acked 1002 Write 10 Sent Done Done false
87762024-02-06T22:19:23.707Z 4 Acked 1003 Read 7 Sent Done Done false
87772024-02-06T22:19:23.707Z 5 Acked 1004 Read 6 Sent Done Done false
87782024-02-06T22:19:23.707Z 6 Acked 1005 Read 9 Sent Done Done false
87792024-02-06T22:19:23.707Z 7 Acked 1006 Read 7 Sent Done Done false
87802024-02-06T22:19:23.707Z 8 Acked 1007 Read 7 Sent Done Done false
87812024-02-06T22:19:23.707Z 9 Acked 1008 Read 5 Sent Done Done false
87822024-02-06T22:19:23.707Z 10 Acked 1009 Read 7 Sent Done Done false
87832024-02-06T22:19:23.707Z 11 Acked 1010 Read 10 Sent Done Done false
87842024-02-06T22:19:23.708Z 12 Acked 1011 Write 2 Sent Done Done false
87852024-02-06T22:19:23.708Z 13 Acked 1012 Write 1 Sent Done Done false
87862024-02-06T22:19:23.708Z 14 Acked 1013 Flush 0 Sent Done Done false
87872024-02-06T22:19:23.708Z 15 Acked 1014 Write 3 Sent Done Done false
87882024-02-06T22:19:23.708Z 16 Acked 1015 Read 1 Sent Done Done false
87892024-02-06T22:19:23.708Z 17 Acked 1016 Read 1 Sent Done Done false
87902024-02-06T22:19:23.708Z 18 Acked 1017 Write 8 Sent Done Done false
87912024-02-06T22:19:23.708Z 19 Acked 1018 Write 4 Sent Done Done false
87922024-02-06T22:19:23.708Z 20 Acked 1019 Read 6 Sent Done Done false
87932024-02-06T22:19:23.708Z 21 Acked 1020 Read 6 Sent Done Done false
87942024-02-06T22:19:23.708Z 22 Acked 1021 Read 10 Sent Done Done false
87952024-02-06T22:19:23.708Z 23 Acked 1022 Write 6 Sent Done Done false
87962024-02-06T22:19:23.708Z 24 Acked 1023 Write 10 Sent Done Done false
87972024-02-06T22:19:23.708Z 25 Acked 1024 Read 5 Sent Done Done false
87982024-02-06T22:19:23.708Z 26 Acked 1025 Write 7 Sent Done Done false
87992024-02-06T22:19:23.708Z 27 Acked 1026 Write 8 Sent Done Done false
88002024-02-06T22:19:23.708Z 28 Acked 1027 Write 6 Sent Done Done false
88012024-02-06T22:19:23.708Z 29 Acked 1028 Write 8 Sent Done Done false
88022024-02-06T22:19:23.708Z 30 Acked 1029 Read 7 Sent Done Done false
88032024-02-06T22:19:23.708Z STATES DS:0 DS:1 DS:2 TOTAL
88042024-02-06T22:19:23.708Z New 0 0 0 0
88052024-02-06T22:19:23.708Z Sent 30 0 0 30
88062024-02-06T22:19:23.708Z Done 0 30 30 60
88072024-02-06T22:19:23.708Z Skipped 0 0 0 0
88082024-02-06T22:19:23.709Z Error 0 0 0 0
88092024-02-06T22:19:23.709ZLast Flush: 0 1013 1013
88102024-02-06T22:19:23.709ZDownstairs last five completed:
88112024-02-06T22:19:23.709ZUpstairs last five completed: 30 29 28 27 26
88122024-02-06T22:19:23.709Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:22.919201018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1455}
88132024-02-06T22:19:23.709ZWrote out file "/tmp/test_repair_verify.data"
88142024-02-06T22:19:23.713Z
88152024-02-06T22:19:23.716ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
88162024-02-06T22:19:23.731ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88172024-02-06T22:19:23.731Z 0 000-019 75 75 75 143 143 143 F F F
88182024-02-06T22:19:23.731Z 1 020-039 75 75 75 143 143 143 F F F
88192024-02-06T22:19:23.731Z 2 040-059 79 79 79 148 148 148 F F F
88202024-02-06T22:19:23.731Z 3 060-079 78 78 78 147 147 147 F F F
88212024-02-06T22:19:23.731Z 4 080-099 79 80 80 148 150 150 F F F <---
88222024-02-06T22:19:23.731Z 5 100-119 77 80 80 146 149 149 F F F <---
88232024-02-06T22:19:23.731Z 6 120-139 79 79 79 148 148 148 F F F
88242024-02-06T22:19:23.731Z 7 140-159 79 79 79 148 148 148 F F F
88252024-02-06T22:19:23.731Z 8 160-179 79 79 79 148 148 148 F T T <---
88262024-02-06T22:19:23.734Z 9 180-199 77 77 77 146 146 146 F F F
88272024-02-06T22:19:23.734Z 10 200-219 72 72 72 135 135 135 F F F
88282024-02-06T22:19:23.734Z 11 220-239 79 79 79 148 148 148 F T T <---
88292024-02-06T22:19:23.734Z 12 240-259 76 76 76 145 145 145 F F F
88302024-02-06T22:19:23.735Z 13 260-279 79 79 79 148 148 148 F T T <---
88312024-02-06T22:19:23.735Z 14 280-299 76 76 76 144 144 144 F T T <---
88322024-02-06T22:19:23.735Z 15 300-319 77 77 77 146 146 146 F F F
88332024-02-06T22:19:23.735Z 16 320-339 62 62 62 117 117 117 F F F
88342024-02-06T22:19:23.735Z 17 340-359 77 77 77 146 146 146 F F F
88352024-02-06T22:19:23.735Z 18 360-379 67 67 67 126 126 126 F F F
88362024-02-06T22:19:23.735Z 19 380-399 79 80 80 148 150 150 F T T <---
88372024-02-06T22:19:23.735Z 20 400-419 77 80 80 146 150 150 F F F <---
88382024-02-06T22:19:23.735Z 21 420-439 79 79 79 148 148 148 F T T <---
88392024-02-06T22:19:23.735Z 22 440-459 58 58 58 113 113 113 F T T <---
88402024-02-06T22:19:23.735Z 23 460-479 79 79 79 148 148 148 F T T <---
88412024-02-06T22:19:23.735Z 24 480-499 72 72 72 136 136 136 F F F
88422024-02-06T22:19:23.735Z 25 500-519 79 80 80 148 150 150 F F F <---
88432024-02-06T22:19:23.735Z 26 520-539 79 79 79 148 148 148 F F F
88442024-02-06T22:19:23.735Z 27 540-559 79 79 79 148 148 148 F F F
88452024-02-06T22:19:23.735Z 28 560-579 79 79 79 148 148 148 F F F
88462024-02-06T22:19:23.735Z 29 580-599 66 66 66 121 121 121 F T T <---
88472024-02-06T22:19:23.735ZMax gen: 80, Max flush: 150
88482024-02-06T22:19:23.735ZError: Difference in extent metadata found!
88492024-02-06T22:19:23.735ZA Difference in extent metadata is expected here
88502024-02-06T22:19:23.735ZOn loop 40
88512024-02-06T22:19:23.735Z
88522024-02-06T22:19:23.738ZVerifying data now
88532024-02-06T22:19:28.381ZNo change after verify
88542024-02-06T22:19:28.381ZLoop: 40 Downstairs dump after verify (and repair):
88552024-02-06T22:19:28.397ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88562024-02-06T22:19:28.397Z 0 000-019 75 75 75 143 143 143 F F F
88572024-02-06T22:19:28.397Z 1 020-039 75 75 75 143 143 143 F F F
88582024-02-06T22:19:28.397Z 2 040-059 79 79 79 148 148 148 F F F
88592024-02-06T22:19:28.398Z 3 060-079 78 78 78 147 147 147 F F F
88602024-02-06T22:19:28.398Z 4 080-099 80 80 80 150 150 150 F F F
88612024-02-06T22:19:28.398Z 5 100-119 80 80 80 149 149 149 F F F
88622024-02-06T22:19:28.398Z 6 120-139 79 79 79 148 148 148 F F F
88632024-02-06T22:19:28.398Z 7 140-159 79 79 79 148 148 148 F F F
88642024-02-06T22:19:28.398Z 8 160-179 81 81 81 151 151 151 F F F
88652024-02-06T22:19:28.398Z 9 180-199 77 77 77 146 146 146 F F F
88662024-02-06T22:19:28.398Z 10 200-219 72 72 72 135 135 135 F F F
88672024-02-06T22:19:28.401Z 11 220-239 81 81 81 151 151 151 F F F
88682024-02-06T22:19:28.401Z 12 240-259 76 76 76 145 145 145 F F F
88692024-02-06T22:19:28.401Z 13 260-279 81 81 81 151 151 151 F F F
88702024-02-06T22:19:28.401Z 14 280-299 81 81 81 151 151 151 F F F
88712024-02-06T22:19:28.401Z 15 300-319 77 77 77 146 146 146 F F F
88722024-02-06T22:19:28.401Z 16 320-339 62 62 62 117 117 117 F F F
88732024-02-06T22:19:28.402Z 17 340-359 77 77 77 146 146 146 F F F
88742024-02-06T22:19:28.402Z 18 360-379 67 67 67 126 126 126 F F F
88752024-02-06T22:19:28.402Z 19 380-399 81 81 81 151 151 151 F F F
88762024-02-06T22:19:28.402Z 20 400-419 80 80 80 150 150 150 F F F
88772024-02-06T22:19:28.402Z 21 420-439 81 81 81 151 151 151 F F F
88782024-02-06T22:19:28.402Z 22 440-459 81 81 81 151 151 151 F F F
88792024-02-06T22:19:28.402Z 23 460-479 81 81 81 151 151 151 F F F
88802024-02-06T22:19:28.402Z 24 480-499 72 72 72 136 136 136 F F F
88812024-02-06T22:19:28.402Z 25 500-519 80 80 80 150 150 150 F F F
88822024-02-06T22:19:28.402Z 26 520-539 79 79 79 148 148 148 F F F
88832024-02-06T22:19:28.402Z 27 540-559 79 79 79 148 148 148 F F F
88842024-02-06T22:19:28.402Z 28 560-579 79 79 79 148 148 148 F F F
88852024-02-06T22:19:28.402Z 29 580-599 81 81 81 151 151 151 F F F
88862024-02-06T22:19:28.402ZMax gen: 81, Max flush: 151
88872024-02-06T22:19:28.402Z
88882024-02-06T22:19:28.402ZBegin loop 41
88892024-02-06T22:19:28.402ZDownstairs to restart: 0
88902024-02-06T22:19:28.418Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630105998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463}
88912024-02-06T22:19:28.423Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
88922024-02-06T22:19:28.423Z,"time":"2024-02-06T22:19:27.630461199Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463}
88932024-02-06T22:19:28.423Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630498584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463}
88942024-02-06T22:19:28.424Z{"msg":"Crucible 1623270b-38cd-4cbe-b384-4f1b6fec169f has session id: 9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.63052814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
88952024-02-06T22:19:28.424Z{"msg":"Upstairs opts: Upstairs UUID: 1623270b-38cd-4cbe-b384-4f1b6fec169f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630575804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
88962024-02-06T22:19:28.424Z{"msg":"Crucible stats registered with UUID: 1623270b-38cd-4cbe-b384-4f1b6fec169f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.63060929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
88972024-02-06T22:19:28.424Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630652444Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463}
88982024-02-06T22:19:28.424Z{"msg":"1623270b-38cd-4cbe-b384-4f1b6fec169f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630786595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
88992024-02-06T22:19:28.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630831569Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89002024-02-06T22:19:28.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630859546Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89012024-02-06T22:19:28.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630884812Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89022024-02-06T22:19:28.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630910379Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89032024-02-06T22:19:28.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.630931896Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89042024-02-06T22:19:28.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631008216Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89052024-02-06T22:19:28.424Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.63104833Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89062024-02-06T22:19:28.424Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631158935Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89072024-02-06T22:19:28.424Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631215757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89082024-02-06T22:19:28.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631259122Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89092024-02-06T22:19:28.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631284008Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89102024-02-06T22:19:28.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631299676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89112024-02-06T22:19:28.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631804258Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89122024-02-06T22:19:28.424Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631842493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89132024-02-06T22:19:28.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631859121Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89142024-02-06T22:19:28.424Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.631874168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89152024-02-06T22:19:28.424Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.6324591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89162024-02-06T22:19:28.424Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.632498284Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89172024-02-06T22:19:28.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633022963Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89182024-02-06T22:19:28.425Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633070057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89192024-02-06T22:19:28.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633099023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89202024-02-06T22:19:28.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633129729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89212024-02-06T22:19:28.425Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633173623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89222024-02-06T22:19:28.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.6331959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89232024-02-06T22:19:28.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633331572Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89242024-02-06T22:19:28.425Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633363247Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89252024-02-06T22:19:28.425Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.633820605Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89262024-02-06T22:19:28.429Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634266755Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89272024-02-06T22:19:28.429Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634310429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89282024-02-06T22:19:28.430Z{"msg":"[0]R flush_numbers[0..12]: [143, 143, 148, 147, 150, 149, 148, 148, 151, 146, 135, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634327627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89292024-02-06T22:19:28.430Z{"msg":"[0]R generation[0..12]: [75, 75, 79, 78, 80, 80, 79, 79, 81, 77, 72, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634342355Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89302024-02-06T22:19:28.430Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634365622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89312024-02-06T22:19:28.430Z{"msg":"[1]R flush_numbers[0..12]: [143, 143, 148, 147, 150, 149, 148, 148, 151, 146, 135, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.63438102Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89322024-02-06T22:19:28.430Z{"msg":"[1]R generation[0..12]: [75, 75, 79, 78, 80, 80, 79, 79, 81, 77, 72, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634402547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89332024-02-06T22:19:28.430Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634418055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89342024-02-06T22:19:28.430Z{"msg":"[2]R flush_numbers[0..12]: [143, 143, 148, 147, 150, 149, 148, 148, 151, 146, 135, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634432473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89352024-02-06T22:19:28.430Z{"msg":"[2]R generation[0..12]: [75, 75, 79, 78, 80, 80, 79, 79, 81, 77, 72, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.63445395Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"Wait for a query_work_queue command to finish before sending IO
89362024-02-06T22:19:28.430Z}
89372024-02-06T22:19:28.430Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634471528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89382024-02-06T22:19:28.430Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634492285Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89392024-02-06T22:19:28.430Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634506323Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89402024-02-06T22:19:28.430Z{"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634546227Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89412024-02-06T22:19:28.430Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634559926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89422024-02-06T22:19:28.430Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634573394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89432024-02-06T22:19:28.430Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634592201Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89442024-02-06T22:19:28.430Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634605919Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89452024-02-06T22:19:28.430Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634620057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89462024-02-06T22:19:28.430Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634641475Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89472024-02-06T22:19:28.430Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634654663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89482024-02-06T22:19:28.430Z{"msg":"1623270b-38cd-4cbe-b384-4f1b6fec169f is now active with session: 9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634668791Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89492024-02-06T22:19:28.430Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634688478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463,"session_id":"9e9b02f5-64b6-4bed-b4b2-deac5d9b1caf"}
89502024-02-06T22:19:28.430Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:27.634702306Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463}
89512024-02-06T22:19:28.430ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
89522024-02-06T22:19:28.431ZLoading write count information from file "/tmp/test_repair_verify.data"
89532024-02-06T22:19:28.431ZRun Repair workload
89542024-02-06T22:19:28.431Z01/30 Flush
89552024-02-06T22:19:28.431Z02/30 Write block 391 len 16384 data: 7 6 6 5
89562024-02-06T22:19:28.431Z03/30 Write block 241 len 28672 data: 5 6 5 5 5 5 9
89572024-02-06T22:19:28.431Z04/30 Flush
89582024-02-06T22:19:28.434Z05/30 Write block 410 len 40960 data: 9 10 7 6 7 9 10 13 12 11
89592024-02-06T22:19:28.434Z06/30 Write block 362 len 16384 data: 12 11 9 8
89602024-02-06T22:19:28.434Z07/30 Write block 12 len 4096 data: 5
89612024-02-06T22:19:28.434Z08/30 Read block 513 len 36864
89622024-02-06T22:19:28.434Z09/30 Write block 45 len 12288 data: 5 6 6
89632024-02-06T22:19:28.434Z10/30 Read block 171 len 16384
89642024-02-06T22:19:28.434Z11/30 Write block 63 len 4096 data: 6
89652024-02-06T22:19:28.434Z12/30 Write block 166 len 36864 data: 4 6 8 10 9 6 9 11 10
89662024-02-06T22:19:28.437Z13/30 Read block 316 len 8192
89672024-02-06T22:19:28.438Z14/30 Read block 545 len 16384
89682024-02-06T22:19:28.438Z15/30 Read block 334 len 36864
89692024-02-06T22:19:28.441Z16/30 Write block 415 len 24576 data: 10 11 14 13 12 10
89702024-02-06T22:19:28.441Z17/30 Read block 164 len 36864
89712024-02-06T22:19:28.441Z18/30 Read block 250 len 24576
89722024-02-06T22:19:28.441Z19/30 Write block 115 len 4096 data: 10
89732024-02-06T22:19:28.444Z20/30 Write block 474 len 28672 data: 8 8 8 9 9 7 4
89742024-02-06T22:19:28.444Z21/30 Read block 274 len 4096
89752024-02-06T22:19:28.444Z22/30 Write block 319 len 32768 data: 7 5 5 5 4 4 5 4
89762024-02-06T22:19:28.444Z23/30 Write block 51 len 32768 data: 12 12 14 16 15 15 12 10
89772024-02-06T22:19:28.447Z24/30 Write block 355 len 24576 data: 13 11 13 11 11 10
89782024-02-06T22:19:28.447Z25/30 Flush
89792024-02-06T22:19:28.451Z26/30 Write block 51 len 24576 data: 13 13 15 17 16 16
89802024-02-06T22:19:28.454Z27/30 Write block 367 len 12288 data: 9 8 7
89812024-02-06T22:19:28.454Z28/30 Read block 199 len 4096
89822024-02-06T22:19:28.454Z29/30 Write block 333 len 24576 data: 6 9 9 10 9 10
89832024-02-06T22:19:28.454Z30/30 Write block 148 len 16384 data: 12 11 9 7
89842024-02-06T22:19:28.458Z----------------------------------------------------------------
89852024-02-06T22:19:28.458Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89862024-02-06T22:19:28.458ZStates: Active Active Active
89872024-02-06T22:19:28.458ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89882024-02-06T22:19:28.458Z 1 Acked 1000 Flush 0 Sent Done Done false
89892024-02-06T22:19:28.458Z 2 Acked 1001 Write 4 Sent Done Done false
89902024-02-06T22:19:28.458Z 3 Acked 1002 Write 7 Sent Done Done false
89912024-02-06T22:19:28.458Z 4 Acked 1003 Flush 0 Sent Done Done false
89922024-02-06T22:19:28.458Z 5 Acked 1004 Write 10 Sent Done Done false
89932024-02-06T22:19:28.458Z 6 Acked 1005 Write 4 Sent Done Done false
89942024-02-06T22:19:28.458Z 7 Acked 1006 Write 1 Sent Done Done false
89952024-02-06T22:19:28.458Z 8 Acked 1007 Read 9 Sent Done Done false
89962024-02-06T22:19:28.458Z 9 Acked 1008 Write 3 Sent Done Done false
89972024-02-06T22:19:28.458Z 10 Acked 1009 Read 4 Sent Done Done false
89982024-02-06T22:19:28.458Z 11 Acked 1010 Write 1 Sent Done Done false
89992024-02-06T22:19:28.458Z 12 Acked 1011 Write 9 Sent Done Done false
90002024-02-06T22:19:28.458Z 13 Acked 1012 Read 2 Sent Done Done false
90012024-02-06T22:19:28.458Z 14 Acked 1013 Read 4 Sent Done Done false
90022024-02-06T22:19:28.458Z 15 Acked 1014 Read 9 Sent Done Done false
90032024-02-06T22:19:28.458Z 16 Acked 1015 Write 6 Sent Done Done false
90042024-02-06T22:19:28.458Z 17 Acked 1016 Read 9 Sent Done Done false
90052024-02-06T22:19:28.459Z 18 Acked 1017 Read 6 Sent Done Done false
90062024-02-06T22:19:28.459Z 19 Acked 1018 Write 1 Sent Done Done false
90072024-02-06T22:19:28.459Z 20 Acked 1019 Write 7 Sent Done Done false
90082024-02-06T22:19:28.459Z 21 Acked 1020 Read 1 Sent Done Done false
90092024-02-06T22:19:28.459Z 22 Acked 1021 Write 8 Sent Done Done false
90102024-02-06T22:19:28.459Z 23 Acked 1022 Write 8 Sent Done Done false
90112024-02-06T22:19:28.459Z 24 Acked 1023 Write 6 Sent Done Done false
90122024-02-06T22:19:28.459Z 25 Acked 1024 Flush 0 Sent Done Done false
90132024-02-06T22:19:28.459Z 26 Acked 1025 Write 6 Sent Done Done false
90142024-02-06T22:19:28.459Z 27 Acked 1026 Write 3 Sent Done Done false
90152024-02-06T22:19:28.459Z 28 Acked 1027 Read 1 Sent Done Done false
90162024-02-06T22:19:28.459Z 29 Acked 1028 Write 6 Sent Sent Done false
90172024-02-06T22:19:28.459Z 30 Acked 1029 Write 4 Sent Sent Sent false
90182024-02-06T22:19:28.459Z STATES DS:0 DS:1 DS:2 TOTAL
90192024-02-06T22:19:28.459Z New 0 0 0 0
90202024-02-06T22:19:28.459Z Sent 30 2 1 33
90212024-02-06T22:19:28.459Z Done 0 28 29 57
90222024-02-06T22:19:28.459Z Skipped 0 0 0 0
90232024-02-06T22:19:28.459Z Error 0 0 0 0
90242024-02-06T22:19:28.459ZLast Flush: 0 1024 1024
90252024-02-06T22:19:28.459ZDownstairs last five completed:
90262024-02-06T22:19:28.459ZUpstairs last five completed: 30 29 28 27 26
90272024-02-06T22:19:28.459Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:27.667485707Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1463}
90282024-02-06T22:19:28.459ZWrote out file "/tmp/test_repair_verify.data"
90292024-02-06T22:19:28.463Z
90302024-02-06T22:19:28.463ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
90312024-02-06T22:19:28.478ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90322024-02-06T22:19:28.478Z 0 000-019 75 82 82 143 154 154 F F F <---
90332024-02-06T22:19:28.478Z 1 020-039 75 75 75 143 143 143 F F F
90342024-02-06T22:19:28.478Z 2 040-059 79 82 82 148 154 154 F T T <---
90352024-02-06T22:19:28.478Z 3 060-079 78 82 82 147 154 154 F F F <---
90362024-02-06T22:19:28.482Z 4 080-099 80 80 80 150 150 150 F F F
90372024-02-06T22:19:28.482Z 5 100-119 80 82 82 149 154 154 F F F <---
90382024-02-06T22:19:28.482Z 6 120-139 79 79 79 148 148 148 F F F
90392024-02-06T22:19:28.482Z 7 140-159 79 79 79 148 148 148 F T T <---
90402024-02-06T22:19:28.482Z 8 160-179 81 82 82 151 154 154 F F F <---
90412024-02-06T22:19:28.482Z 9 180-199 77 77 77 146 146 146 F F F
90422024-02-06T22:19:28.483Z 10 200-219 72 72 72 135 135 135 F F F
90432024-02-06T22:19:28.483Z 11 220-239 81 81 81 151 151 151 F F F
90442024-02-06T22:19:28.483Z 12 240-259 76 82 82 145 153 153 F F F <---
90452024-02-06T22:19:28.483Z 13 260-279 81 81 81 151 151 151 F F F
90462024-02-06T22:19:28.483Z 14 280-299 81 81 81 151 151 151 F F F
90472024-02-06T22:19:28.483Z 15 300-319 77 82 82 146 154 154 F F F <---
90482024-02-06T22:19:28.483Z 16 320-339 62 82 82 117 154 154 F T T <---
90492024-02-06T22:19:28.483Z 17 340-359 77 82 82 146 154 154 F F F <---
90502024-02-06T22:19:28.483Z 18 360-379 67 82 82 126 154 154 F T T <---
90512024-02-06T22:19:28.483Z 19 380-399 81 82 82 151 153 153 F F F <---
90522024-02-06T22:19:28.483Z 20 400-419 80 82 82 150 154 154 F F F <---
90532024-02-06T22:19:28.483Z 21 420-439 81 82 82 151 154 154 F F F <---
90542024-02-06T22:19:28.483Z 22 440-459 81 81 81 151 151 151 F F F
90552024-02-06T22:19:28.483Z 23 460-479 81 82 82 151 154 154 F F F <---
90562024-02-06T22:19:28.483Z 24 480-499 72 82 82 136 154 154 F F F <---
90572024-02-06T22:19:28.483Z 25 500-519 80 80 80 150 150 150 F F F
90582024-02-06T22:19:28.483Z 26 520-539 79 79 79 148 148 148 F F F
90592024-02-06T22:19:28.483Z 27 540-559 79 79 79 148 148 148 F F F
90602024-02-06T22:19:28.483Z 28 560-579 79 79 79 148 148 148 F F F
90612024-02-06T22:19:28.483Z 29 580-599 81 81 81 151 151 151 F F F
90622024-02-06T22:19:28.483ZMax gen: 82, Max flush: 154
90632024-02-06T22:19:28.483ZError: Difference in extent metadata found!
90642024-02-06T22:19:28.483ZA Difference in extent metadata is expected here
90652024-02-06T22:19:28.484ZOn loop 41
90662024-02-06T22:19:28.484Z
90672024-02-06T22:19:28.489ZVerifying data now
90682024-02-06T22:19:33.239ZNo change after verify
90692024-02-06T22:19:33.239ZLoop: 41 Downstairs dump after verify (and repair):
90702024-02-06T22:19:33.254ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90712024-02-06T22:19:33.254Z 0 000-019 82 82 82 154 154 154 F F F
90722024-02-06T22:19:33.254Z 1 020-039 75 75 75 143 143 143 F F F
90732024-02-06T22:19:33.254Z 2 040-059 83 83 83 155 155 155 F F F
90742024-02-06T22:19:33.254Z 3 060-079 82 82 82 154 154 154 F F F
90752024-02-06T22:19:33.254Z 4 080-099 80 80 80 150 150 150 F F F
90762024-02-06T22:19:33.255Z 5 100-119 82 82 82 154 154 154 F F F
90772024-02-06T22:19:33.255Z 6 120-139 79 79 79 148 148 148 F F F
90782024-02-06T22:19:33.255Z 7 140-159 83 83 83 155 155 155 F F F
90792024-02-06T22:19:33.255Z 8 160-179 82 82 82 154 154 154 F F F
90802024-02-06T22:19:33.258Z 9 180-199 77 77 77 146 146 146 F F F
90812024-02-06T22:19:33.258Z 10 200-219 72 72 72 135 135 135 F F F
90822024-02-06T22:19:33.258Z 11 220-239 81 81 81 151 151 151 F F F
90832024-02-06T22:19:33.258Z 12 240-259 82 82 82 153 153 153 F F F
90842024-02-06T22:19:33.258Z 13 260-279 81 81 81 151 151 151 F F F
90852024-02-06T22:19:33.258Z 14 280-299 81 81 81 151 151 151 F F F
90862024-02-06T22:19:33.259Z 15 300-319 82 82 82 154 154 154 F F F
90872024-02-06T22:19:33.259Z 16 320-339 83 83 83 155 155 155 F F F
90882024-02-06T22:19:33.259Z 17 340-359 82 82 82 154 154 154 F F F
90892024-02-06T22:19:33.259Z 18 360-379 83 83 83 155 155 155 F F F
90902024-02-06T22:19:33.259Z 19 380-399 82 82 82 153 153 153 F F F
90912024-02-06T22:19:33.259Z 20 400-419 82 82 82 154 154 154 F F F
90922024-02-06T22:19:33.259Z 21 420-439 82 82 82 154 154 154 F F F
90932024-02-06T22:19:33.259Z 22 440-459 81 81 81 151 151 151 F F F
90942024-02-06T22:19:33.259Z 23 460-479 82 82 82 154 154 154 F F F
90952024-02-06T22:19:33.259Z 24 480-499 82 82 82 154 154 154 F F F
90962024-02-06T22:19:33.259Z 25 500-519 80 80 80 150 150 150 F F F
90972024-02-06T22:19:33.259Z 26 520-539 79 79 79 148 148 148 F F F
90982024-02-06T22:19:33.259Z 27 540-559 79 79 79 148 148 148 F F F
90992024-02-06T22:19:33.259Z 28 560-579 79 79 79 148 148 148 F F F
91002024-02-06T22:19:33.259Z 29 580-599 81 81 81 151 151 151 F F F
91012024-02-06T22:19:33.259ZMax gen: 83, Max flush: 155
91022024-02-06T22:19:33.259Z
91032024-02-06T22:19:33.259ZBegin loop 42
91042024-02-06T22:19:33.259ZDownstairs to restart: 0
91052024-02-06T22:19:33.275Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486139558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471}
91062024-02-06T22:19:33.275Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486402393Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471}
91072024-02-06T22:19:33.275Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible"Crucible runtime is spawned
91082024-02-06T22:19:33.287Z,"level":30,"time":"2024-02-06T22:19:32.486453616Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471}
91092024-02-06T22:19:33.287Z{"msg":"Crucible 779ad50e-751e-42d9-aa67-3ef9628c144e has session id: 21e45bfe-eb6f-488e-8515-c1878578b8df","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.48649642Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91102024-02-06T22:19:33.287Z{"msg":"Upstairs opts: Upstairs UUID: 779ad50e-751e-42d9-aa67-3ef9628c144e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486558792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91112024-02-06T22:19:33.287Z{"msg":"Crucible stats registered with UUID: 779ad50e-751e-42d9-aa67-3ef9628c144e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486604435Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91122024-02-06T22:19:33.287Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486635361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471}
91132024-02-06T22:19:33.287Z{"msg":"779ad50e-751e-42d9-aa67-3ef9628c144e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486684655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91142024-02-06T22:19:33.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486729499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91152024-02-06T22:19:33.287Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486777812Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91162024-02-06T22:19:33.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486823776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91172024-02-06T22:19:33.287Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.48687157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91182024-02-06T22:19:33.287Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.486934871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91192024-02-06T22:19:33.287Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487020569Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91202024-02-06T22:19:33.287Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487056745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91212024-02-06T22:19:33.287Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487131854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91222024-02-06T22:19:33.287Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.48716558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91232024-02-06T22:19:33.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487244609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91242024-02-06T22:19:33.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487281524Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91252024-02-06T22:19:33.288Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487302431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91262024-02-06T22:19:33.288Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487853447Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91272024-02-06T22:19:33.288Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487895391Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91282024-02-06T22:19:33.288Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487922577Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91292024-02-06T22:19:33.288Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.487961562Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91302024-02-06T22:19:33.288Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.488450576Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91312024-02-06T22:19:33.288Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.488563411Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91322024-02-06T22:19:33.288Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.488816227Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91332024-02-06T22:19:33.288Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.48886671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91342024-02-06T22:19:33.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.488965996Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91352024-02-06T22:19:33.288Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.489036647Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91362024-02-06T22:19:33.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.489074762Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91372024-02-06T22:19:33.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.489099529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91382024-02-06T22:19:33.288Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.489133924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91392024-02-06T22:19:33.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.489158481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91402024-02-06T22:19:33.288Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.48945701Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91412024-02-06T22:19:33.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.489914008Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91422024-02-06T22:19:33.288Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490000937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91432024-02-06T22:19:33.288Z{"msg":"[0]R flush_numbers[0..12]: [154, 143, 155, 154, 150, 154, 148, 155, 154, 146, 135, 151]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
91442024-02-06T22:19:33.288Z,"time":"2024-02-06T22:19:32.490037982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91452024-02-06T22:19:33.289Z{"msg":"[0]R generation[0..12]: [82, 75, 83, 82, 80, 82, 79, 83, 82, 77, 72, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490055109Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91462024-02-06T22:19:33.289Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490078316Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91472024-02-06T22:19:33.289Z{"msg":"[1]R flush_numbers[0..12]: [154, 143, 155, 154, 150, 154, 148, 155, 154, 146, 135, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490093104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91482024-02-06T22:19:33.289Z{"msg":"[1]R generation[0..12]: [82, 75, 83, 82, 80, 82, 79, 83, 82, 77, 72, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490108322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91492024-02-06T22:19:33.289Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490139408Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91502024-02-06T22:19:33.289Z{"msg":"[2]R flush_numbers[0..12]: [154, 143, 155, 154, 150, 154, 148, 155, 154, 146, 135, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490158385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91512024-02-06T22:19:33.289Z{"msg":"[2]R generation[0..12]: [82, 75, 83, 82, 80, 82, 79, 83, 82, 77, 72, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490180552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91522024-02-06T22:19:33.289Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490203009Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91532024-02-06T22:19:33.289Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490226056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91542024-02-06T22:19:33.289Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490243904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91552024-02-06T22:19:33.289Z{"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490264321Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91562024-02-06T22:19:33.289Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490278079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91572024-02-06T22:19:33.289Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490297976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91582024-02-06T22:19:33.289Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490310725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91592024-02-06T22:19:33.289Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490330692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91602024-02-06T22:19:33.289Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.4903443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91612024-02-06T22:19:33.289Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490364367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91622024-02-06T22:19:33.289Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490377036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91632024-02-06T22:19:33.289Z{"msg":"779ad50e-751e-42d9-aa67-3ef9628c144e is now active with session: 21e45bfe-eb6f-488e-8515-c1878578b8df","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490390004Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91642024-02-06T22:19:33.289Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.490408572Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471,"session_id":"21e45bfe-eb6f-488e-8515-c1878578b8df"}
91652024-02-06T22:19:33.289Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:32.49042149Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471}
91662024-02-06T22:19:33.289ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91672024-02-06T22:19:33.289ZLoading write count information from file "/tmp/test_repair_verify.data"
91682024-02-06T22:19:33.290ZRun Repair workload
91692024-02-06T22:19:33.290Z01/30 Flush
91702024-02-06T22:19:33.290Z02/30 Write block 179 len 24576 data: 12 12 12 10 11 13
91712024-02-06T22:19:33.290Z03/30 Read block 199 len 40960
91722024-02-06T22:19:33.290Z04/30 Read block 438 len 36864
91732024-02-06T22:19:33.294Z05/30 Read block 90 len 20480
91742024-02-06T22:19:33.294Z06/30 Read block 557 len 36864
91752024-02-06T22:19:33.294Z07/30 Flush
91762024-02-06T22:19:33.295Z08/30 Write block 201 len 32768 data: 5 8 8 8 7 9 8 8
91772024-02-06T22:19:33.295Z09/30 Write block 385 len 24576 data: 9 9 7 6 6 6
91782024-02-06T22:19:33.295Z10/30 Read block 15 len 4096
91792024-02-06T22:19:33.295Z11/30 Write block 508 len 8192 data: 8 6
91802024-02-06T22:19:33.295Z12/30 Write block 484 len 4096 data: 6
91812024-02-06T22:19:33.295Z13/30 Read block 143 len 20480
91822024-02-06T22:19:33.295Z14/30 Read block 102 len 4096
91832024-02-06T22:19:33.295Z15/30 Write block 348 len 8192 data: 12 11
91842024-02-06T22:19:33.295Z16/30 Write block 301 len 20480 data: 8 7 6 5 8
91852024-02-06T22:19:33.298Z17/30 Read block 274 len 32768
91862024-02-06T22:19:33.298Z18/30 Read block 343 len 4096
91872024-02-06T22:19:33.298Z19/30 Read block 11 len 4096
91882024-02-06T22:19:33.298Z20/30 Read block 442 len 12288
91892024-02-06T22:19:33.298Z21/30 Read block 111 len 20480
91902024-02-06T22:19:33.299Z22/30 Write block 388 len 20480 data: 7 7 7 8 7
91912024-02-06T22:19:33.299Z23/30 Read block 575 len 32768
91922024-02-06T22:19:33.303Z24/30 Read block 315 len 24576
91932024-02-06T22:19:33.303Z25/30 Read block 543 len 16384
91942024-02-06T22:19:33.303Z26/30 Write block 84 len 8192 data: 6 5
91952024-02-06T22:19:33.303Z27/30 Read block 359 len 16384
91962024-02-06T22:19:33.303Z28/30 Read block 143 len 16384
91972024-02-06T22:19:33.306Z29/30 Write block 174 len 8192 data: 11 9
91982024-02-06T22:19:33.306Z30/30 Write block 289 len 12288 data: 8 11 11
91992024-02-06T22:19:33.306Z----------------------------------------------------------------
92002024-02-06T22:19:33.306Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:23
92012024-02-06T22:19:33.306ZStates: Active Active Active
92022024-02-06T22:19:33.306ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
92032024-02-06T22:19:33.306Z 8 Acked 1007 Write 8 Done Done Done false
92042024-02-06T22:19:33.306Z 9 Acked 1008 Write 6 Done Done Done false
92052024-02-06T22:19:33.306Z 10 Acked 1009 Read 1 Done Done Done false
92062024-02-06T22:19:33.307Z 11 Acked 1010 Write 2 Done Done Done false
92072024-02-06T22:19:33.307Z 12 Acked 1011 Write 1 Done Done Done false
92082024-02-06T22:19:33.307Z 13 Acked 1012 Read 5 Done Done Done false
92092024-02-06T22:19:33.307Z 14 Acked 1013 Read 1 Sent Done Done false
92102024-02-06T22:19:33.307Z 15 Acked 1014 Write 2 Sent Done Done false
92112024-02-06T22:19:33.307Z 16 Acked 1015 Write 5 Sent Done Done false
92122024-02-06T22:19:33.307Z 17 Acked 1016 Read 8 Sent Done Done false
92132024-02-06T22:19:33.307Z 18 Acked 1017 Read 1 Sent Done Done false
92142024-02-06T22:19:33.307Z 19 Acked 1018 Read 1 Sent Done Done false
92152024-02-06T22:19:33.307Z 20 Acked 1019 Read 3 Sent Done Done false
92162024-02-06T22:19:33.307Z 21 Acked 1020 Read 5 Sent Done Done false
92172024-02-06T22:19:33.307Z 22 Acked 1021 Write 5 Sent Done Done false
92182024-02-06T22:19:33.307Z 23 Acked 1022 Read 8 Sent Done Done false
92192024-02-06T22:19:33.307Z 24 Acked 1023 Read 6 Sent Done Done false
92202024-02-06T22:19:33.307Z 25 Acked 1024 Read 4 Sent Done Done false
92212024-02-06T22:19:33.307Z 26 Acked 1025 Write 2 Sent Done Done false
92222024-02-06T22:19:33.307Z 27 Acked 1026 Read 4 Sent Done Done false
92232024-02-06T22:19:33.307Z 28 Acked 1027 Read 4 Sent Done Done false
92242024-02-06T22:19:33.307Z 29 Acked 1028 Write 2 Sent Done Done false
92252024-02-06T22:19:33.307Z 30 Acked 1029 Write 3 Sent Sent Sent false
92262024-02-06T22:19:33.307Z STATES DS:0 DS:1 DS:2 TOTAL
92272024-02-06T22:19:33.307Z New 0 0 0 0
92282024-02-06T22:19:33.307Z Sent 17 1 1 19
92292024-02-06T22:19:33.307Z Done 6 22 22 50
92302024-02-06T22:19:33.308Z Skipped 0 0 0 0
92312024-02-06T22:19:33.308Z Error 0 0 0 0
92322024-02-06T22:19:33.308ZLast Flush: 1006 1006 1006
92332024-02-06T22:19:33.308ZDownstairs last five completed: 1006 1005 1004 1003 1002
92342024-02-06T22:19:33.308ZUpstairs last five completed: 30 29 28 27 26
92352024-02-06T22:19:33.308Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:32.516353886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1471}
92362024-02-06T22:19:33.308ZWrote out file "/tmp/test_repair_verify.data"
92372024-02-06T22:19:33.311Z
92382024-02-06T22:19:33.314ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
92392024-02-06T22:19:33.331ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92402024-02-06T22:19:33.331Z 0 000-019 82 82 82 154 154 154 F F F
92412024-02-06T22:19:33.331Z 1 020-039 75 75 75 143 143 143 F F F
92422024-02-06T22:19:33.331Z 2 040-059 83 83 83 155 155 155 F F F
92432024-02-06T22:19:33.331Z 3 060-079 82 82 82 154 154 154 F F F
92442024-02-06T22:19:33.331Z 4 080-099 80 80 80 150 150 150 F T T <---
92452024-02-06T22:19:33.331Z 5 100-119 82 82 82 154 154 154 F F F
92462024-02-06T22:19:33.331Z 6 120-139 79 79 79 148 148 148 F F F
92472024-02-06T22:19:33.331Z 7 140-159 83 83 83 155 155 155 F F F
92482024-02-06T22:19:33.331Z 8 160-179 84 84 84 157 157 157 F T T <---
92492024-02-06T22:19:33.331Z 9 180-199 84 84 84 157 157 157 F F F
92502024-02-06T22:19:33.335Z 10 200-219 72 72 72 135 135 135 T T T
92512024-02-06T22:19:33.335Z 11 220-239 81 81 81 151 151 151 F F F
92522024-02-06T22:19:33.335Z 12 240-259 82 82 82 153 153 153 F F F
92532024-02-06T22:19:33.335Z 13 260-279 81 81 81 151 151 151 F F F
92542024-02-06T22:19:33.335Z 14 280-299 81 81 81 151 151 151 F T T <---
92552024-02-06T22:19:33.335Z 15 300-319 82 82 82 154 154 154 F T T <---
92562024-02-06T22:19:33.335Z 16 320-339 83 83 83 155 155 155 F F F
92572024-02-06T22:19:33.335Z 17 340-359 82 82 82 154 154 154 F T T <---
92582024-02-06T22:19:33.335Z 18 360-379 83 83 83 155 155 155 F F F
92592024-02-06T22:19:33.336Z 19 380-399 82 82 82 153 153 153 T T T
92602024-02-06T22:19:33.336Z 20 400-419 82 82 82 154 154 154 F F F
92612024-02-06T22:19:33.336Z 21 420-439 82 82 82 154 154 154 F F F
92622024-02-06T22:19:33.336Z 22 440-459 81 81 81 151 151 151 F F F
92632024-02-06T22:19:33.336Z 23 460-479 82 82 82 154 154 154 F F F
92642024-02-06T22:19:33.336Z 24 480-499 82 82 82 154 154 154 T T T
92652024-02-06T22:19:33.336Z 25 500-519 80 80 80 150 150 150 T T T
92662024-02-06T22:19:33.336Z 26 520-539 79 79 79 148 148 148 F F F
92672024-02-06T22:19:33.336Z 27 540-559 79 79 79 148 148 148 F F F
92682024-02-06T22:19:33.336Z 28 560-579 79 79 79 148 148 148 F F F
92692024-02-06T22:19:33.336Z 29 580-599 81 81 81 151 151 151 F F F
92702024-02-06T22:19:33.336ZMax gen: 84, Max flush: 157
92712024-02-06T22:19:33.336ZError: Difference in extent metadata found!
92722024-02-06T22:19:33.336ZA Difference in extent metadata is expected here
92732024-02-06T22:19:33.336ZOn loop 42
92742024-02-06T22:19:33.336Z
92752024-02-06T22:19:33.339ZVerifying data now
92762024-02-06T22:19:34.866ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
92772024-02-06T22:19:34.879Z391,393d390
92782024-02-06T22:19:34.879Z< 6,
92792024-02-06T22:19:34.879Z< 6,
92802024-02-06T22:19:34.879Z< 6,
92812024-02-06T22:19:34.879Z394a392,395
92822024-02-06T22:19:34.879Z> 7,
92832024-02-06T22:19:34.880Z> 7,
92842024-02-06T22:19:34.880Z> 8,
92852024-02-06T22:19:34.880Z> 7,
92862024-02-06T22:19:34.880Z396d396
92872024-02-06T22:19:34.880Z< 6,
92882024-02-06T22:19:34.880Zdiff found after verify
92892024-02-06T22:19:34.880ZLoop: 42 Downstairs dump after verify (and repair):
92902024-02-06T22:19:34.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92912024-02-06T22:19:34.886Z 0 000-019 82 82 82 154 154 154 F F F
92922024-02-06T22:19:34.886Z 1 020-039 75 75 75 143 143 143 F F F
92932024-02-06T22:19:34.886Z 2 040-059 83 83 83 155 155 155 F F F
92942024-02-06T22:19:34.886Z 3 060-079 82 82 82 154 154 154 F F F
92952024-02-06T22:19:34.886Z 4 080-099 85 85 85 158 158 158 F F F
92962024-02-06T22:19:34.886Z 5 100-119 82 82 82 154 154 154 F F F
92972024-02-06T22:19:34.886Z 6 120-139 79 79 79 148 148 148 F F F
92982024-02-06T22:19:34.886Z 7 140-159 83 83 83 155 155 155 F F F
92992024-02-06T22:19:34.890Z 8 160-179 85 85 85 158 158 158 F F F
93002024-02-06T22:19:34.890Z 9 180-199 84 84 84 157 157 157 F F F
93012024-02-06T22:19:34.890Z 10 200-219 85 85 85 158 158 158 F F F
93022024-02-06T22:19:34.890Z 11 220-239 81 81 81 151 151 151 F F F
93032024-02-06T22:19:34.890Z 12 240-259 82 82 82 153 153 153 F F F
93042024-02-06T22:19:34.890Z 13 260-279 81 81 81 151 151 151 F F F
93052024-02-06T22:19:34.890Z 14 280-299 85 85 85 158 158 158 F F F
93062024-02-06T22:19:34.890Z 15 300-319 85 85 85 158 158 158 F F F
93072024-02-06T22:19:34.890Z 16 320-339 83 83 83 155 155 155 F F F
93082024-02-06T22:19:34.890Z 17 340-359 85 85 85 158 158 158 F F F
93092024-02-06T22:19:34.890Z 18 360-379 83 83 83 155 155 155 F F F
93102024-02-06T22:19:34.890Z 19 380-399 85 85 85 158 158 158 F F F
93112024-02-06T22:19:34.890Z 20 400-419 82 82 82 154 154 154 F F F
93122024-02-06T22:19:34.890Z 21 420-439 82 82 82 154 154 154 F F F
93132024-02-06T22:19:34.890Z 22 440-459 81 81 81 151 151 151 F F F
93142024-02-06T22:19:34.890Z 23 460-479 82 82 82 154 154 154 F F F
93152024-02-06T22:19:34.890Z 24 480-499 85 85 85 158 158 158 F F F
93162024-02-06T22:19:34.890Z 25 500-519 85 85 85 158 158 158 F F F
93172024-02-06T22:19:34.890Z 26 520-539 79 79 79 148 148 148 F F F
93182024-02-06T22:19:34.890Z 27 540-559 79 79 79 148 148 148 F F F
93192024-02-06T22:19:34.890Z 28 560-579 79 79 79 148 148 148 F F F
93202024-02-06T22:19:34.891Z 29 580-599 81 81 81 151 151 151 F F F
93212024-02-06T22:19:34.891ZMax gen: 85, Max flush: 158
93222024-02-06T22:19:34.891Z
93232024-02-06T22:19:34.891ZBegin loop 43
93242024-02-06T22:19:34.891ZDownstairs to restart: 2
93252024-02-06T22:19:34.910Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.121383746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482}
93262024-02-06T22:19:34.917Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.121646611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482}
93272024-02-06T22:19:34.917Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.121692774Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482}
93282024-02-06T22:19:34.917Z{"msg":"Crucible cd5cdf29-4473-4e47-b981-88b0d3f3d155 has session id: c17ec59f-50b3-4f76-af15-9a0c296d09d1","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
93292024-02-06T22:19:34.917Z2024-02-06T22:19:34.12172361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93302024-02-06T22:19:34.918Z{"msg":"Upstairs opts: Upstairs UUID: cd5cdf29-4473-4e47-b981-88b0d3f3d155, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.121792771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93312024-02-06T22:19:34.918Z{"msg":"Crucible stats registered with UUID: cd5cdf29-4473-4e47-b981-88b0d3f3d155","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.121819547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93322024-02-06T22:19:34.918Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.121851853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482}
93332024-02-06T22:19:34.918Z{"msg":"cd5cdf29-4473-4e47-b981-88b0d3f3d155 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.121989744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93342024-02-06T22:19:34.918Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.12202311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93352024-02-06T22:19:34.918Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122041207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93362024-02-06T22:19:34.918Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122074803Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93372024-02-06T22:19:34.918Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122102969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93382024-02-06T22:19:34.918Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122141594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93392024-02-06T22:19:34.918Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122178229Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93402024-02-06T22:19:34.918Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122225073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93412024-02-06T22:19:34.918Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122264957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93422024-02-06T22:19:34.918Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122304112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93432024-02-06T22:19:34.918Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:34.122334788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93442024-02-06T22:19:34.918Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:34.122371223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93452024-02-06T22:19:34.918Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122387561Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93462024-02-06T22:19:34.918Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122401449Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93472024-02-06T22:19:34.918Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:34.122424906Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93482024-02-06T22:19:34.918Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122437804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93492024-02-06T22:19:34.918Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122452232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93502024-02-06T22:19:34.918Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122544079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93512024-02-06T22:19:34.918Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122927648Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93522024-02-06T22:19:34.918Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.122970272Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93532024-02-06T22:19:34.919Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.123009487Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93542024-02-06T22:19:34.919Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.123050071Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93552024-02-06T22:19:34.919Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.123610725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93562024-02-06T22:19:34.919Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.123664948Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93572024-02-06T22:19:34.919Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.124138784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93582024-02-06T22:19:34.919Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.124189627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93592024-02-06T22:19:34.919Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.124207705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93602024-02-06T22:19:34.919Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.124222453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93612024-02-06T22:19:34.919Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.124236781Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93622024-02-06T22:19:34.919Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:34.124260867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93632024-02-06T22:19:35.913Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.123828333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93642024-02-06T22:19:35.918Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.124047564Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93652024-02-06T22:19:35.918Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.125052668Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93662024-02-06T22:19:35.918Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.12510944Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93672024-02-06T22:19:35.918Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.125594315Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93682024-02-06T22:19:35.918Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126108005Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93692024-02-06T22:19:35.918Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126134072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93702024-02-06T22:19:35.918Z{"msg":"[0]R flush_numbers[0..12]: [154, 143, 155, 154, 158, 154, 148, 155, 158, 157, 158, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126159508Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93712024-02-06T22:19:35.918Z{"msg":"[0]R generation[0..12]: [82, 75, 83, 82, 85, 82, 79, 83, 85, 84, 85, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126175356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93722024-02-06T22:19:35.918Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126198853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93732024-02-06T22:19:35.918Z{"msg":"[1]R flush_numbers[0..12]: [154, 143, 155, 154, 158, 154, 148, 155, 158, 157, 158, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126214761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93742024-02-06T22:19:35.918Z{"msg":"[1]R generation[0..12]: [82, 75, 83, 82, 85, 82, 79, 83, 85, 84, 85, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126235858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93752024-02-06T22:19:35.918Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126250446Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93762024-02-06T22:19:35.919Z{"msg":"[2]R flush_numbers[0..12]: [154, 143, 155, 154, 158, 154, 148, 155, 158, 157, 158, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126271093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"Wait for a query_work_queue command to finish before sending IO
93772024-02-06T22:19:35.919Zdownstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93782024-02-06T22:19:35.919Z{"msg":"[2]R generation[0..12]: [82, 75, 83, 82, 85, 82, 79, 83, 85, 84, 85, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126288601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93792024-02-06T22:19:35.919Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126310478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93802024-02-06T22:19:35.919Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126324326Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93812024-02-06T22:19:35.919Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126338424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93822024-02-06T22:19:35.919Z{"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126358431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93832024-02-06T22:19:35.919Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.12637236Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93842024-02-06T22:19:35.919Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126393926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93852024-02-06T22:19:35.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126406625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93862024-02-06T22:19:35.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126419983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93872024-02-06T22:19:35.919Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126433101Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93882024-02-06T22:19:35.920Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126446579Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93892024-02-06T22:19:35.920Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126459018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93902024-02-06T22:19:35.920Z{"msg":"cd5cdf29-4473-4e47-b981-88b0d3f3d155 is now active with session: c17ec59f-50b3-4f76-af15-9a0c296d09d1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126472056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93912024-02-06T22:19:35.920Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126484074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482,"session_id":"c17ec59f-50b3-4f76-af15-9a0c296d09d1"}
93922024-02-06T22:19:35.920Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:35.126497283Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482}
93932024-02-06T22:19:35.920ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93942024-02-06T22:19:35.920ZLoading write count information from file "/tmp/test_repair_verify.data"
93952024-02-06T22:19:35.920ZRun Repair workload
93962024-02-06T22:19:35.926Z01/30 Write block 98 len 24576 data: 6 6 6 3 4 4
93972024-02-06T22:19:35.926Z02/30 Read block 504 len 20480
93982024-02-06T22:19:35.926Z03/30 Read block 407 len 8192
93992024-02-06T22:19:35.926Z04/30 Read block 246 len 16384
94002024-02-06T22:19:35.926Z05/30 Read block 114 len 36864
94012024-02-06T22:19:35.926Z06/30 Read block 548 len 16384
94022024-02-06T22:19:35.926Z07/30 Write block 584 len 8192 data: 5 7
94032024-02-06T22:19:35.926Z08/30 Write block 509 len 40960 data: 7 7 7 6 9 9 8 8 8 7
94042024-02-06T22:19:35.926Z09/30 Read block 122 len 24576
94052024-02-06T22:19:35.930Z10/30 Write block 368 len 8192 data: 9 8
94062024-02-06T22:19:35.930Z11/30 Write block 138 len 12288 data: 7 7 5
94072024-02-06T22:19:35.930Z12/30 Read block 298 len 16384
94082024-02-06T22:19:35.930Z13/30 Read block 257 len 32768
94092024-02-06T22:19:35.930Z14/30 Read block 309 len 24576
94102024-02-06T22:19:35.930Z15/30 Read block 103 len 32768
94112024-02-06T22:19:35.933Z16/30 Read block 296 len 24576
94122024-02-06T22:19:35.933Z17/30 Write block 544 len 40960 data: 8 6 7 5 5 6 5 6 6 6
94132024-02-06T22:19:35.933Z18/30 Write block 356 len 40960 data: 12 14 12 12 11 9 13 12 10 9
94142024-02-06T22:19:35.934Z19/30 Flush
94152024-02-06T22:19:35.937Z20/30 Write block 264 len 40960 data: 6 7 9 9 9 8 7 6 7 5
94162024-02-06T22:19:35.941Z21/30 Read block 102 len 32768
94172024-02-06T22:19:35.941Z22/30 Read block 440 len 40960
94182024-02-06T22:19:35.945Z23/30 Flush
94192024-02-06T22:19:35.945Z24/30 Write block 513 len 24576 data: 10 10 9 9 9 8
94202024-02-06T22:19:35.945Z25/30 Flush
94212024-02-06T22:19:35.945Z26/30 Write block 286 len 24576 data: 4 4 8 9 12 12
94222024-02-06T22:19:35.960Z27/30 Write block 37 len 16384 data: 5 5 6 9
94232024-02-06T22:19:35.960Z28/30 Read block 503 len 4096
94242024-02-06T22:19:35.960Z29/30 Read block 422 len 4096
94252024-02-06T22:19:35.960Z30/30 Write block 573 len 40960 data: 10 11 10 11 12 11 10 9 6 5
94262024-02-06T22:19:35.960Z----------------------------------------------------------------
94272024-02-06T22:19:35.960Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
94282024-02-06T22:19:35.961ZStates: Active Active Active
94292024-02-06T22:19:35.961ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
94302024-02-06T22:19:35.961Z 1 Acked 1000 Write 6 Done Done Done false
94312024-02-06T22:19:35.961Z 2 Acked 1001 Read 5 Done Done Done false
94322024-02-06T22:19:35.961Z 3 Acked 1002 Read 2 Done Done Done false
94332024-02-06T22:19:35.961Z 4 Acked 1003 Read 4 Done Done Done false
94342024-02-06T22:19:35.961Z 5 Acked 1004 Read 9 Done Done Done false
94352024-02-06T22:19:35.961Z 6 Acked 1005 Read 4 Done Done Done false
94362024-02-06T22:19:35.961Z 7 Acked 1006 Write 2 Done Done Done false
94372024-02-06T22:19:35.961Z 8 Acked 1007 Write 10 Done Done Done false
94382024-02-06T22:19:35.961Z 9 Acked 1008 Read 6 Done Done Done false
94392024-02-06T22:19:35.961Z 10 Acked 1009 Write 2 Done Done Done false
94402024-02-06T22:19:35.962Z 11 Acked 1010 Write 3 Done Done Done false
94412024-02-06T22:19:35.962Z 12 Acked 1011 Read 4 Done Done Sent false
94422024-02-06T22:19:35.962Z 13 Acked 1012 Read 8 Done Done Sent false
94432024-02-06T22:19:35.962Z 14 Acked 1013 Read 6 Done Done Sent false
94442024-02-06T22:19:35.962Z 15 Acked 1014 Read 8 Done Done Sent false
94452024-02-06T22:19:35.962Z 16 Acked 1015 Read 6 Done Done Sent false
94462024-02-06T22:19:35.962Z 17 Acked 1016 Write 10 Done Done Sent false
94472024-02-06T22:19:35.962Z 18 Acked 1017 Write 10 Done Done Sent false
94482024-02-06T22:19:35.962Z 19 Acked 1018 Flush 0 Done Done Sent false
94492024-02-06T22:19:35.962Z 20 Acked 1019 Write 10 Done Done Sent false
94502024-02-06T22:19:35.962Z 21 Acked 1020 Read 8 Done Done Sent false
94512024-02-06T22:19:35.962Z 22 Acked 1021 Read 10 Done Done Sent false
94522024-02-06T22:19:35.962Z 23 Acked 1022 Flush 0 Done Done Sent false
94532024-02-06T22:19:35.963Z 24 Acked 1023 Write 6 Done Done Sent false
94542024-02-06T22:19:35.963Z 25 Acked 1024 Flush 0 Done Done Sent false
94552024-02-06T22:19:35.963Z 26 Acked 1025 Write 6 Done Done Sent false
94562024-02-06T22:19:35.963Z 27 Acked 1026 Write 4 Done Done Sent false
94572024-02-06T22:19:35.963Z 28 Acked 1027 Read 1 Done Done Sent false
94582024-02-06T22:19:35.963Z 29 Acked 1028 Read 1 Done Done Sent false
94592024-02-06T22:19:35.963Z 30 Acked 1029 Write 10 Sent Sent Sent false
94602024-02-06T22:19:35.963Z STATES DS:0 DS:1 DS:2 TOTAL
94612024-02-06T22:19:35.963Z New 0 0 0 0
94622024-02-06T22:19:35.963Z Sent 1 1 19 21
94632024-02-06T22:19:35.963Z Done 29 29 11 69
94642024-02-06T22:19:35.963Z Skipped 0 0 0 0
94652024-02-06T22:19:35.963Z Error 0 0 0 0
94662024-02-06T22:19:35.964ZLast Flush: 1024 1024 0
94672024-02-06T22:19:35.964ZDownstairs last five completed:
94682024-02-06T22:19:35.964ZUpstairs last five completed: 30 29 28 27 26
94692024-02-06T22:19:35.964Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:35.159333146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1482}
94702024-02-06T22:19:35.964ZWrote out file "/tmp/test_repair_verify.data"
94712024-02-06T22:19:35.968Z
94722024-02-06T22:19:35.968ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94732024-02-06T22:19:35.974ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94742024-02-06T22:19:35.975Z 0 000-019 82 82 82 154 154 154 F F F
94752024-02-06T22:19:35.975Z 1 020-039 75 75 75 143 143 143 T T F <---
94762024-02-06T22:19:35.975Z 2 040-059 83 83 83 155 155 155 T T F <---
94772024-02-06T22:19:35.975Z 3 060-079 82 82 82 154 154 154 F F F
94782024-02-06T22:19:35.975Z 4 080-099 86 86 85 159 159 158 F F T <---
94792024-02-06T22:19:35.975Z 5 100-119 86 86 82 159 159 154 F F T <---
94802024-02-06T22:19:35.975Z 6 120-139 86 86 79 159 159 148 F F T <---
94812024-02-06T22:19:35.975Z 7 140-159 86 86 83 159 159 155 F F T <---
94822024-02-06T22:19:35.979Z 8 160-179 85 85 85 158 158 158 F F F
94832024-02-06T22:19:35.979Z 9 180-199 84 84 84 157 157 157 F F F
94842024-02-06T22:19:35.980Z 10 200-219 85 85 85 158 158 158 F F F
94852024-02-06T22:19:35.980Z 11 220-239 81 81 81 151 151 151 F F F
94862024-02-06T22:19:35.980Z 12 240-259 82 82 82 153 153 153 F F F
94872024-02-06T22:19:35.980Z 13 260-279 86 86 81 160 160 151 F F F <---
94882024-02-06T22:19:35.980Z 14 280-299 85 85 85 158 158 158 T T F <---
94892024-02-06T22:19:35.980Z 15 300-319 85 85 85 158 158 158 F F F
94902024-02-06T22:19:35.980Z 16 320-339 83 83 83 155 155 155 F F F
94912024-02-06T22:19:35.980Z 17 340-359 86 86 85 159 159 158 F F F <---
94922024-02-06T22:19:35.980Z 18 360-379 86 86 83 159 159 155 F F T <---
94932024-02-06T22:19:35.980Z 19 380-399 85 85 85 158 158 158 F F F
94942024-02-06T22:19:35.980Z 20 400-419 82 82 82 154 154 154 F F F
94952024-02-06T22:19:35.980Z 21 420-439 82 82 82 154 154 154 F F F
94962024-02-06T22:19:35.980Z 22 440-459 81 81 81 151 151 151 F F F
94972024-02-06T22:19:35.980Z 23 460-479 82 82 82 154 154 154 F F F
94982024-02-06T22:19:35.980Z 24 480-499 85 85 85 158 158 158 F F F
94992024-02-06T22:19:35.980Z 25 500-519 86 86 85 161 161 158 F F T <---
95002024-02-06T22:19:35.980Z 26 520-539 79 79 79 148 148 148 F F F
95012024-02-06T22:19:35.980Z 27 540-559 86 86 79 159 159 148 F F F <---
95022024-02-06T22:19:35.980Z 28 560-579 79 79 79 148 148 148 T T F <---
95032024-02-06T22:19:35.980Z 29 580-599 86 86 81 159 159 151 T T T <---
95042024-02-06T22:19:35.980ZMax gen: 86, Max flush: 161
95052024-02-06T22:19:35.980ZError: Difference in extent metadata found!
95062024-02-06T22:19:35.980ZA Difference in extent metadata is expected here
95072024-02-06T22:19:35.980ZOn loop 43
95082024-02-06T22:19:35.980Z
95092024-02-06T22:19:35.985ZVerifying data now
95102024-02-06T22:19:41.680ZNo change after verify
95112024-02-06T22:19:41.683ZLoop: 43 Downstairs dump after verify (and repair):
95122024-02-06T22:19:41.703ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95132024-02-06T22:19:41.703Z 0 000-019 82 82 82 154 154 154 F F F
95142024-02-06T22:19:41.703Z 1 020-039 87 87 87 162 162 162 F F F
95152024-02-06T22:19:41.703Z 2 040-059 87 87 87 162 162 162 F F F
95162024-02-06T22:19:41.703Z 3 060-079 82 82 82 154 154 154 F F F
95172024-02-06T22:19:41.703Z 4 080-099 86 86 86 159 159 159 F F F
95182024-02-06T22:19:41.704Z 5 100-119 86 86 86 159 159 159 F F F
95192024-02-06T22:19:41.704Z 6 120-139 86 86 86 159 159 159 F F F
95202024-02-06T22:19:41.704Z 7 140-159 86 86 86 159 159 159 F F F
95212024-02-06T22:19:41.704Z 8 160-179 85 85 85 158 158 158 F F F
95222024-02-06T22:19:41.704Z 9 180-199 84 84 84 157 157 157 F F F
95232024-02-06T22:19:41.704Z 10 200-219 85 85 85 158 158 158 F F F
95242024-02-06T22:19:41.717Z 11 220-239 81 81 81 151 151 151 F F F
95252024-02-06T22:19:41.717Z 12 240-259 82 82 82 153 153 153 F F F
95262024-02-06T22:19:41.717Z 13 260-279 86 86 86 160 160 160 F F F
95272024-02-06T22:19:41.717Z 14 280-299 87 87 87 162 162 162 F F F
95282024-02-06T22:19:41.717Z 15 300-319 85 85 85 158 158 158 F F F
95292024-02-06T22:19:41.717Z 16 320-339 83 83 83 155 155 155 F F F
95302024-02-06T22:19:41.717Z 17 340-359 86 86 86 159 159 159 F F F
95312024-02-06T22:19:41.717Z 18 360-379 86 86 86 159 159 159 F F F
95322024-02-06T22:19:41.717Z 19 380-399 85 85 85 158 158 158 F F F
95332024-02-06T22:19:41.717Z 20 400-419 82 82 82 154 154 154 F F F
95342024-02-06T22:19:41.717Z 21 420-439 82 82 82 154 154 154 F F F
95352024-02-06T22:19:41.717Z 22 440-459 81 81 81 151 151 151 F F F
95362024-02-06T22:19:41.717Z 23 460-479 82 82 82 154 154 154 F F F
95372024-02-06T22:19:41.717Z 24 480-499 85 85 85 158 158 158 F F F
95382024-02-06T22:19:41.717Z 25 500-519 86 86 86 161 161 161 F F F
95392024-02-06T22:19:41.717Z 26 520-539 79 79 79 148 148 148 F F F
95402024-02-06T22:19:41.717Z 27 540-559 86 86 86 159 159 159 F F F
95412024-02-06T22:19:41.717Z 28 560-579 87 87 87 162 162 162 F F F
95422024-02-06T22:19:41.718Z 29 580-599 87 87 87 162 162 162 F F F
95432024-02-06T22:19:41.718ZMax gen: 87, Max flush: 162
95442024-02-06T22:19:41.718Z
95452024-02-06T22:19:41.718ZBegin loop 44
95462024-02-06T22:19:41.718ZDownstairs to restart: 1
95472024-02-06T22:19:41.723Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.927968713Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490}
95482024-02-06T22:19:41.723Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928263503Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490}
95492024-02-06T22:19:41.723Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
95502024-02-06T22:19:41.723Z,"time":"2024-02-06T22:19:40.928296689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490}
95512024-02-06T22:19:41.723Z{"msg":"Crucible bb6f9b0a-adfd-4786-bbc7-9aec776d92e4 has session id: 2ca98ac6-87d1-4c81-a662-71af2e44eb56","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928323825Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95522024-02-06T22:19:41.723Z{"msg":"Upstairs opts: Upstairs UUID: bb6f9b0a-adfd-4786-bbc7-9aec776d92e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928367259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95532024-02-06T22:19:41.723Z{"msg":"Crucible stats registered with UUID: bb6f9b0a-adfd-4786-bbc7-9aec776d92e4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928402055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95542024-02-06T22:19:41.723Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928425771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490}
95552024-02-06T22:19:41.723Z{"msg":"bb6f9b0a-adfd-4786-bbc7-9aec776d92e4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928481294Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95562024-02-06T22:19:41.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928517479Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95572024-02-06T22:19:41.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928554404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95582024-02-06T22:19:41.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.92858465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95592024-02-06T22:19:41.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928628104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95602024-02-06T22:19:41.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928682886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95612024-02-06T22:19:41.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928724721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95622024-02-06T22:19:41.723Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928819358Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95632024-02-06T22:19:41.724Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.92887732Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95642024-02-06T22:19:41.724Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928902567Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95652024-02-06T22:19:41.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928941312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95662024-02-06T22:19:41.724Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.928990595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95672024-02-06T22:19:41.724Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:40.929022101Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95682024-02-06T22:19:41.724Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:40.929066975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95692024-02-06T22:19:41.724Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:40.929095311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95702024-02-06T22:19:41.724Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.929128496Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95712024-02-06T22:19:41.724Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.929154183Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95722024-02-06T22:19:41.724Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.929187578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95732024-02-06T22:19:41.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.929456782Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95742024-02-06T22:19:41.724Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.929490997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95752024-02-06T22:19:41.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.929506165Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95762024-02-06T22:19:41.724Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.929528462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95772024-02-06T22:19:41.724Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930111853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95782024-02-06T22:19:41.724Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930157727Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95792024-02-06T22:19:41.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930682406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95802024-02-06T22:19:41.724Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930719881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95812024-02-06T22:19:41.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930735559Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95822024-02-06T22:19:41.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930748418Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95832024-02-06T22:19:41.724Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930770044Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95842024-02-06T22:19:41.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:40.930783873Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95852024-02-06T22:19:42.720Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.930209028Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95862024-02-06T22:19:42.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.930580188Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95872024-02-06T22:19:42.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.931373001Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95882024-02-06T22:19:42.725Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.931412015Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95892024-02-06T22:19:42.725Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.931862604Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95902024-02-06T22:19:42.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932369666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95912024-02-06T22:19:42.725Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932410261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95922024-02-06T22:19:42.725Z{"msg":"[0]R flush_numbers[0..12]: [154, 162, 162, 154, 159, 159, 159, 159, 158, 157, 158, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932429518Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95932024-02-06T22:19:42.725Z{"msg":"[0]R generation[0..12]: [82, 87, 87, 82, 86, 86, 86, 86, 85, 84, 85, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932448325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95942024-02-06T22:19:42.725Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932474632Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95952024-02-06T22:19:42.725Z{"msg":"[1]R flush_numbers[0..12]: [154, 162, 162, 154, 159, 159, 159, 159, 158, 157, 158, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932492749Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95962024-02-06T22:19:42.725Z{"msg":"[1]R generation[0..12]: [82, 87, 87, 82, 86, 86, 86, 86, 85, 84, 85, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932510047Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95972024-02-06T22:19:42.725Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932536624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
95982024-02-06T22:19:42.725Z{"msg":"[2]R flush_numbers[0..12]: [154, 162, 162, 154, 159, 159, 159, 159, 158, 157, 158, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932554881Z","hostname":Wait for a query_work_queue command to finish before sending IO
95992024-02-06T22:19:42.725Z"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96002024-02-06T22:19:42.725Z{"msg":"[2]R generation[0..12]: [82, 87, 87, 82, 86, 86, 86, 86, 85, 84, 85, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932588306Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96012024-02-06T22:19:42.725Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932607154Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96022024-02-06T22:19:42.725Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932632631Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96032024-02-06T22:19:42.725Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932650048Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96042024-02-06T22:19:42.725Z{"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932676614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96052024-02-06T22:19:42.725Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932703361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96062024-02-06T22:19:42.725Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932721759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96072024-02-06T22:19:42.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932737196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96082024-02-06T22:19:42.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932753894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96092024-02-06T22:19:42.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932799278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96102024-02-06T22:19:42.726Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932816746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96112024-02-06T22:19:42.726Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932841982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96122024-02-06T22:19:42.726Z{"msg":"bb6f9b0a-adfd-4786-bbc7-9aec776d92e4 is now active with session: 2ca98ac6-87d1-4c81-a662-71af2e44eb56","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.93285894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96132024-02-06T22:19:42.726Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932882737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490,"session_id":"2ca98ac6-87d1-4c81-a662-71af2e44eb56"}
96142024-02-06T22:19:42.726Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:41.932898844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490}
96152024-02-06T22:19:42.726ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
96162024-02-06T22:19:42.726ZLoading write count information from file "/tmp/test_repair_verify.data"
96172024-02-06T22:19:42.726ZRun Repair workload
96182024-02-06T22:19:42.729Z01/30 Write block 368 len 16384 data: 10 9 7 7
96192024-02-06T22:19:42.729Z02/30 Read block 16 len 16384
96202024-02-06T22:19:42.730Z03/30 Read block 371 len 12288
96212024-02-06T22:19:42.730Z04/30 Read block 258 len 20480
96222024-02-06T22:19:42.730Z05/30 Flush
96232024-02-06T22:19:42.733Z06/30 Write block 193 len 16384 data: 6 8 8 11
96242024-02-06T22:19:42.733Z07/30 Write block 211 len 8192 data: 9 8
96252024-02-06T22:19:42.733Z08/30 Read block 246 len 16384
96262024-02-06T22:19:42.733Z09/30 Write block 153 len 32768 data: 10 9 10 8 10 8 8 7
96272024-02-06T22:19:42.733Z10/30 Write block 175 len 20480 data: 10 9 8 8 13
96282024-02-06T22:19:42.736Z11/30 Flush
96292024-02-06T22:19:42.736Z12/30 Flush
96302024-02-06T22:19:42.739Z13/30 Write block 492 len 32768 data: 4 5 6 6 6 6 7 7
96312024-02-06T22:19:42.739Z14/30 Read block 524 len 8192
96322024-02-06T22:19:42.739Z15/30 Write block 110 len 12288 data: 7 5 7
96332024-02-06T22:19:42.742Z16/30 Write block 73 len 24576 data: 6 8 7 4 5 4
96342024-02-06T22:19:42.742Z17/30 Write block 7 len 36864 data: 8 7 6 8 5 6 4 6 7
96352024-02-06T22:19:42.742Z18/30 Read block 150 len 32768
96362024-02-06T22:19:42.742Z19/30 Read block 120 len 28672
96372024-02-06T22:19:42.745Z20/30 Write block 129 len 28672 data: 10 11 10 8 8 7 6
96382024-02-06T22:19:42.745Z21/30 Write block 462 len 24576 data: 4 4 4 4 3 7
96392024-02-06T22:19:42.745Z22/30 Flush
96402024-02-06T22:19:42.748Z23/30 Write block 543 len 20480 data: 7 9 7 8 6
96412024-02-06T22:19:42.751Z24/30 Write block 392 len 24576 data: 7 7 6 6 5 6
96422024-02-06T22:19:42.751Z25/30 Read block 85 len 32768
96432024-02-06T22:19:42.751Z26/30 Write block 222 len 24576 data: 10 10 10 8 8 6
96442024-02-06T22:19:42.754Z27/30 Read block 288 len 36864
96452024-02-06T22:19:42.754Z28/30 Read block 220 len 4096
96462024-02-06T22:19:42.754Z29/30 Read block 57 len 8192
96472024-02-06T22:19:42.754Z30/30 Read block 198 len 16384
96482024-02-06T22:19:42.757Z----------------------------------------------------------------
96492024-02-06T22:19:42.757Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
96502024-02-06T22:19:42.758ZStates: Active Active Active
96512024-02-06T22:19:42.758ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
96522024-02-06T22:19:42.758Z 1 Acked 1000 Write 4 Done Done Done false
96532024-02-06T22:19:42.758Z 2 Acked 1001 Read 4 Done Done Done false
96542024-02-06T22:19:42.758Z 3 Acked 1002 Read 3 Done Done Done false
96552024-02-06T22:19:42.758Z 4 Acked 1003 Read 5 Done Sent Done false
96562024-02-06T22:19:42.758Z 5 Acked 1004 Flush 0 Done Sent Done false
96572024-02-06T22:19:42.758Z 6 Acked 1005 Write 4 Done Sent Done false
96582024-02-06T22:19:42.758Z 7 Acked 1006 Write 2 Done Sent Done false
96592024-02-06T22:19:42.758Z 8 Acked 1007 Read 4 Done Sent Done false
96602024-02-06T22:19:42.758Z 9 Acked 1008 Write 8 Done Sent Done false
96612024-02-06T22:19:42.758Z 10 Acked 1009 Write 5 Done Sent Done false
96622024-02-06T22:19:42.758Z 11 Acked 1010 Flush 0 Done Sent Done false
96632024-02-06T22:19:42.758Z 12 Acked 1011 Flush 0 Done Sent Done false
96642024-02-06T22:19:42.758Z 13 Acked 1012 Write 8 Done Sent Done false
96652024-02-06T22:19:42.758Z 14 Acked 1013 Read 2 Done Sent Done false
96662024-02-06T22:19:42.758Z 15 Acked 1014 Write 3 Done Sent Done false
96672024-02-06T22:19:42.758Z 16 Acked 1015 Write 6 Done Sent Done false
96682024-02-06T22:19:42.758Z 17 Acked 1016 Write 9 Done Sent Done false
96692024-02-06T22:19:42.758Z 18 Acked 1017 Read 8 Done Sent Done false
96702024-02-06T22:19:42.758Z 19 Acked 1018 Read 7 Done Sent Done false
96712024-02-06T22:19:42.758Z 20 Acked 1019 Write 7 Done Sent Done false
96722024-02-06T22:19:42.758Z 21 Acked 1020 Write 6 Done Sent Done false
96732024-02-06T22:19:42.758Z 22 Acked 1021 Flush 0 Done Sent Done false
96742024-02-06T22:19:42.759Z 23 Acked 1022 Write 5 Done Sent Done false
96752024-02-06T22:19:42.759Z 24 Acked 1023 Write 6 Done Sent Done false
96762024-02-06T22:19:42.759Z 25 Acked 1024 Read 8 Done Sent Done false
96772024-02-06T22:19:42.759Z 26 Acked 1025 Write 6 Done Sent Done false
96782024-02-06T22:19:42.759Z 27 Acked 1026 Read 9 Done Sent Done false
96792024-02-06T22:19:42.759Z 28 Acked 1027 Read 1 Done Sent Done false
96802024-02-06T22:19:42.759Z 29 Acked 1028 Read 2 Done Sent Done false
96812024-02-06T22:19:42.759Z 30 Acked 1029 Read 4 Done Sent Done false
96822024-02-06T22:19:42.759Z STATES DS:0 DS:1 DS:2 TOTAL
96832024-02-06T22:19:42.759Z New 0 0 0 0
96842024-02-06T22:19:42.759Z Sent 0 27 0 27
96852024-02-06T22:19:42.759Z Done 30 3 30 63
96862024-02-06T22:19:42.759Z Skipped 0 0 0 0
96872024-02-06T22:19:42.759Z Error 0 0 0 0
96882024-02-06T22:19:42.759ZLast Flush: 1021 0 1021
96892024-02-06T22:19:42.759ZDownstairs last five completed:
96902024-02-06T22:19:42.759ZUpstairs last five completed: 30 29 28 27 26
96912024-02-06T22:19:42.759Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:41.965167775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1490}
96922024-02-06T22:19:42.759ZWrote out file "/tmp/test_repair_verify.data"
96932024-02-06T22:19:42.762Z
96942024-02-06T22:19:42.765ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96952024-02-06T22:19:42.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96962024-02-06T22:19:42.780Z 0 000-019 88 82 88 166 154 166 F F F <---
96972024-02-06T22:19:42.780Z 1 020-039 87 87 87 162 162 162 F F F
96982024-02-06T22:19:42.780Z 2 040-059 87 87 87 162 162 162 F F F
96992024-02-06T22:19:42.780Z 3 060-079 88 82 88 166 154 166 F F F <---
97002024-02-06T22:19:42.780Z 4 080-099 86 86 86 159 159 159 F F F
97012024-02-06T22:19:42.784Z 5 100-119 88 86 88 166 159 166 F F F <---
97022024-02-06T22:19:42.784Z 6 120-139 88 86 88 166 159 166 F F F <---
97032024-02-06T22:19:42.784Z 7 140-159 88 86 88 164 159 164 F F F <---
97042024-02-06T22:19:42.784Z 8 160-179 88 85 88 164 158 164 F F F <---
97052024-02-06T22:19:42.784Z 9 180-199 88 84 88 164 157 164 F F F <---
97062024-02-06T22:19:42.784Z 10 200-219 88 85 88 164 158 164 F F F <---
97072024-02-06T22:19:42.784Z 11 220-239 81 81 81 151 151 151 T F T <---
97082024-02-06T22:19:42.784Z 12 240-259 82 82 82 153 153 153 F F F
97092024-02-06T22:19:42.784Z 13 260-279 86 86 86 160 160 160 F F F
97102024-02-06T22:19:42.784Z 14 280-299 87 87 87 162 162 162 F F F
97112024-02-06T22:19:42.784Z 15 300-319 85 85 85 158 158 158 F F F
97122024-02-06T22:19:42.784Z 16 320-339 83 83 83 155 155 155 F F F
97132024-02-06T22:19:42.784Z 17 340-359 86 86 86 159 159 159 F F F
97142024-02-06T22:19:42.784Z 18 360-379 88 86 88 163 159 163 F T F <---
97152024-02-06T22:19:42.784Z 19 380-399 85 85 85 158 158 158 T F T <---
97162024-02-06T22:19:42.784Z 20 400-419 82 82 82 154 154 154 F F F
97172024-02-06T22:19:42.784Z 21 420-439 82 82 82 154 154 154 F F F
97182024-02-06T22:19:42.785Z 22 440-459 81 81 81 151 151 151 F F F
97192024-02-06T22:19:42.785Z 23 460-479 88 82 88 166 154 166 F F F <---
97202024-02-06T22:19:42.785Z 24 480-499 88 85 88 166 158 166 F F F <---
97212024-02-06T22:19:42.785Z 25 500-519 86 86 86 161 161 161 F F F
97222024-02-06T22:19:42.785Z 26 520-539 79 79 79 148 148 148 F F F
97232024-02-06T22:19:42.785Z 27 540-559 86 86 86 159 159 159 T F T <---
97242024-02-06T22:19:42.785Z 28 560-579 87 87 87 162 162 162 F F F
97252024-02-06T22:19:42.785Z 29 580-599 87 87 87 162 162 162 F F F
97262024-02-06T22:19:42.785ZMax gen: 88, Max flush: 166
97272024-02-06T22:19:42.785ZError: Difference in extent metadata found!
97282024-02-06T22:19:42.785ZA Difference in extent metadata is expected here
97292024-02-06T22:19:42.785ZOn loop 44
97302024-02-06T22:19:42.785Z
97312024-02-06T22:19:42.788ZVerifying data now
97322024-02-06T22:19:48.453ZNo change after verify
97332024-02-06T22:19:48.453ZLoop: 44 Downstairs dump after verify (and repair):
97342024-02-06T22:19:48.468ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97352024-02-06T22:19:48.468Z 0 000-019 88 88 88 166 166 166 F F F
97362024-02-06T22:19:48.468Z 1 020-039 87 87 87 162 162 162 F F F
97372024-02-06T22:19:48.469Z 2 040-059 87 87 87 162 162 162 F F F
97382024-02-06T22:19:48.469Z 3 060-079 88 88 88 166 166 166 F F F
97392024-02-06T22:19:48.469Z 4 080-099 86 86 86 159 159 159 F F F
97402024-02-06T22:19:48.469Z 5 100-119 88 88 88 166 166 166 F F F
97412024-02-06T22:19:48.469Z 6 120-139 88 88 88 166 166 166 F F F
97422024-02-06T22:19:48.469Z 7 140-159 88 88 88 164 164 164 F F F
97432024-02-06T22:19:48.469Z 8 160-179 88 88 88 164 164 164 F F F
97442024-02-06T22:19:48.469Z 9 180-199 88 88 88 164 164 164 F F F
97452024-02-06T22:19:48.474Z 10 200-219 88 88 88 164 164 164 F F F
97462024-02-06T22:19:48.474Z 11 220-239 89 89 89 167 167 167 F F F
97472024-02-06T22:19:48.474Z 12 240-259 82 82 82 153 153 153 F F F
97482024-02-06T22:19:48.474Z 13 260-279 86 86 86 160 160 160 F F F
97492024-02-06T22:19:48.474Z 14 280-299 87 87 87 162 162 162 F F F
97502024-02-06T22:19:48.474Z 15 300-319 85 85 85 158 158 158 F F F
97512024-02-06T22:19:48.474Z 16 320-339 83 83 83 155 155 155 F F F
97522024-02-06T22:19:48.474Z 17 340-359 86 86 86 159 159 159 F F F
97532024-02-06T22:19:48.474Z 18 360-379 88 88 88 163 163 163 F F F
97542024-02-06T22:19:48.474Z 19 380-399 89 89 89 167 167 167 F F F
97552024-02-06T22:19:48.474Z 20 400-419 82 82 82 154 154 154 F F F
97562024-02-06T22:19:48.474Z 21 420-439 82 82 82 154 154 154 F F F
97572024-02-06T22:19:48.474Z 22 440-459 81 81 81 151 151 151 F F F
97582024-02-06T22:19:48.474Z 23 460-479 88 88 88 166 166 166 F F F
97592024-02-06T22:19:48.474Z 24 480-499 88 88 88 166 166 166 F F F
97602024-02-06T22:19:48.474Z 25 500-519 86 86 86 161 161 161 F F F
97612024-02-06T22:19:48.475Z 26 520-539 79 79 79 148 148 148 F F F
97622024-02-06T22:19:48.475Z 27 540-559 89 89 89 167 167 167 F F F
97632024-02-06T22:19:48.475Z 28 560-579 87 87 87 162 162 162 F F F
97642024-02-06T22:19:48.475Z 29 580-599 87 87 87 162 162 162 F F F
97652024-02-06T22:19:48.475ZMax gen: 89, Max flush: 167
97662024-02-06T22:19:48.475Z
97672024-02-06T22:19:48.475ZBegin loop 45
97682024-02-06T22:19:48.475ZDownstairs to restart: 2
97692024-02-06T22:19:48.490Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.698989095Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498}
97702024-02-06T22:19:48.490Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699258789Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498}
97712024-02-06T22:19:48.490Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699279606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498}
97722024-02-06T22:19:48.495Z{"msg":"Crucible 05495de1-8936-490a-aa0b-4db61e74efb1 has session id: c2eb82ae-35b9-4d90-9295-af8d6592357e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699312622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}Crucible runtime is spawned
97732024-02-06T22:19:48.495Z
97742024-02-06T22:19:48.495Z{"msg":"Upstairs opts: Upstairs UUID: 05495de1-8936-490a-aa0b-4db61e74efb1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699372694Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97752024-02-06T22:19:48.495Z{"msg":"Crucible stats registered with UUID: 05495de1-8936-490a-aa0b-4db61e74efb1","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699416158Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97762024-02-06T22:19:48.496Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699433655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498}
97772024-02-06T22:19:48.496Z{"msg":"05495de1-8936-490a-aa0b-4db61e74efb1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699528453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97782024-02-06T22:19:48.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699565218Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97792024-02-06T22:19:48.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699641827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97802024-02-06T22:19:48.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699690571Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97812024-02-06T22:19:48.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699731135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97822024-02-06T22:19:48.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699759931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97832024-02-06T22:19:48.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699783248Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97842024-02-06T22:19:48.496Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.69991485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97852024-02-06T22:19:48.496Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699955175Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97862024-02-06T22:19:48.496Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.699984061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97872024-02-06T22:19:48.497Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.700024016Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97882024-02-06T22:19:48.497Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.700053072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97892024-02-06T22:19:48.497Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.700089777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97902024-02-06T22:19:48.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.700605637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97912024-02-06T22:19:48.497Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.700643212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97922024-02-06T22:19:48.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.70065939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97932024-02-06T22:19:48.497Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.700680517Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97942024-02-06T22:19:48.497Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701218324Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97952024-02-06T22:19:48.497Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701350616Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97962024-02-06T22:19:48.497Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701716107Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97972024-02-06T22:19:48.497Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701773189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97982024-02-06T22:19:48.497Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701810534Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
97992024-02-06T22:19:48.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701825472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98002024-02-06T22:19:48.498Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701855528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98012024-02-06T22:19:48.498Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701936317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98022024-02-06T22:19:48.498Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701964633Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98032024-02-06T22:19:48.498Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.701979311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98042024-02-06T22:19:48.498Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.702349512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98052024-02-06T22:19:48.552Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.702814619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98062024-02-06T22:19:48.552Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.702862272Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98072024-02-06T22:19:48.552Z{"msg":"[0]R flush_numbers[0..12]: [166, 162, 162, 166, 159, 166, 166, 164, 164, 164, 164, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.702900497Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98082024-02-06T22:19:48.552Z{"msg":"[0]R generation[0..12]: [88, 87, 87, 88, 86, 88, 88, 88, 88, 88, 88, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.702936482Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98092024-02-06T22:19:48.552Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.702962289Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98102024-02-06T22:19:48.552Z{"msg":"[1]R flush_numbers[0..12]: [166, 162, 162, 166, 159, 166, 166, 164, 164, 164, 164, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703000133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98112024-02-06T22:19:48.552Z{"msg":"[1]R generation[0..12]: [88, 87, 87, 88, 86, 88, 88, 88, 88, 88, 88, 89]","vWait for a query_work_queue command to finish before sending IO
98122024-02-06T22:19:48.552Z":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703042988Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98132024-02-06T22:19:48.552Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703083052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98142024-02-06T22:19:48.552Z{"msg":"[2]R flush_numbers[0..12]: [166, 162, 162, 166, 159, 166, 166, 164, 164, 164, 164, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703122867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98152024-02-06T22:19:48.552Z{"msg":"[2]R generation[0..12]: [88, 87, 87, 88, 86, 88, 88, 88, 88, 88, 88, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703158562Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98162024-02-06T22:19:48.552Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703198757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98172024-02-06T22:19:48.552Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703239061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98182024-02-06T22:19:48.553Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703277806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98192024-02-06T22:19:48.553Z{"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703339868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98202024-02-06T22:19:48.553Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703387551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98212024-02-06T22:19:48.553Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703427466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98222024-02-06T22:19:48.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703454362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98232024-02-06T22:19:48.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703491597Z"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
98242024-02-06T22:19:48.553Z,"hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98252024-02-06T22:19:48.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703526973Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98262024-02-06T22:19:48.553Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703551859Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98272024-02-06T22:19:48.553Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703565257Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98282024-02-06T22:19:48.553Z{"msg":"05495de1-8936-490a-aa0b-4db61e74efb1 is now active with session: c2eb82ae-35b9-4d90-9295-af8d6592357e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703586554Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98292024-02-06T22:19:48.553Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.703599073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498,"session_id":"c2eb82ae-35b9-4d90-9295-af8d6592357e"}
98302024-02-06T22:19:48.553Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:47.7036192Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498}
98312024-02-06T22:19:48.553ZLoading write count information from file "/tmp/test_repair_verify.data"
98322024-02-06T22:19:48.553ZRun Repair workload
98332024-02-06T22:19:48.553Z01/30 Write block 67 len 16384 data: 6 7 7 7
98342024-02-06T22:19:48.553Z02/30 Write block 484 len 20480 data: 7 6 6 5 4
98352024-02-06T22:19:48.553Z03/30 Read block 531 len 36864
98362024-02-06T22:19:48.553Z04/30 Read block 60 len 32768
98372024-02-06T22:19:48.553Z05/30 Read block 582 len 8192
98382024-02-06T22:19:48.553Z06/30 Read block 309 len 16384
98392024-02-06T22:19:48.553Z07/30 Write block 40 len 8192 data: 10 8
98402024-02-06T22:19:48.553Z08/30 Write block 405 len 4096 data: 6
98412024-02-06T22:19:48.554Z09/30 Flush
98422024-02-06T22:19:48.554Z10/30 Write block 201 len 8192 data: 6 9
98432024-02-06T22:19:48.554Z11/30 Write block 178 len 36864 data: 9 14 13 13 11 12 14 14 14
98442024-02-06T22:19:48.554Z12/30 Write block 260 len 24576 data: 6 9 9 8 7 8
98452024-02-06T22:19:48.554Z13/30 Flush
98462024-02-06T22:19:48.554Z14/30 Write block 307 len 40960 data: 4 5 5 7 8 9 10 10 9 8
98472024-02-06T22:19:48.554Z15/30 Read block 0 len 16384
98482024-02-06T22:19:48.554Z16/30 Write block 556 len 12288 data: 6 6 7
98492024-02-06T22:19:48.554Z17/30 Write block 186 len 36864 data: 15 12 8 7 6 7 7 7 9
98502024-02-06T22:19:48.554Z18/30 Read block 354 len 24576
98512024-02-06T22:19:48.554Z19/30 Read block 203 len 16384
98522024-02-06T22:19:48.554Z20/30 Write block 583 len 40960 data: 5 6 8 5 8 8 10 12 11 10
98532024-02-06T22:19:48.554Z21/30 Write block 276 len 24576 data: 9 9 10 11 11 10
98542024-02-06T22:19:48.554Z22/30 Read block 170 len 4096
98552024-02-06T22:19:48.554Z23/30 Write block 303 len 36864 data: 7 6 9 8 5 6 6 8 9
98562024-02-06T22:19:48.554Z24/30 Write block 24 len 4096 data: 6
98572024-02-06T22:19:48.554Z25/30 Read block 373 len 16384
98582024-02-06T22:19:48.554Z26/30 Flush
98592024-02-06T22:19:48.554Z27/30 Write block 487 len 12288 data: 6 5 5
98602024-02-06T22:19:48.554Z28/30 Read block 578 len 8192
98612024-02-06T22:19:48.554Z29/30 Read block 512 len 12288
98622024-02-06T22:19:48.554Z30/30 Read block 151 len 36864
98632024-02-06T22:19:48.554Z----------------------------------------------------------------
98642024-02-06T22:19:48.555Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
98652024-02-06T22:19:48.555ZStates: Active Active Active
98662024-02-06T22:19:48.555ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
98672024-02-06T22:19:48.555Z 1 Acked 1000 Write 4 Done Done Done false
98682024-02-06T22:19:48.555Z 2 Acked 1001 Write 5 Done Done Done false
98692024-02-06T22:19:48.555Z 3 Acked 1002 Read 9 Done Done Done false
98702024-02-06T22:19:48.555Z 4 Acked 1003 Read 8 Done Done Sent false
98712024-02-06T22:19:48.555Z 5 Acked 1004 Read 2 Done Done Sent false
98722024-02-06T22:19:48.555Z 6 Acked 1005 Read 4 Done Done Sent false
98732024-02-06T22:19:48.555Z 7 Acked 1006 Write 2 Done Done Sent false
98742024-02-06T22:19:48.555Z 8 Acked 1007 Write 1 Done Done Sent false
98752024-02-06T22:19:48.555Z 9 Acked 1008 Flush 0 Done Done Sent false
98762024-02-06T22:19:48.555Z 10 Acked 1009 Write 2 Done Done Sent false
98772024-02-06T22:19:48.555Z 11 Acked 1010 Write 9 Done Done Sent false
98782024-02-06T22:19:48.555Z 12 Acked 1011 Write 6 Done Done Sent false
98792024-02-06T22:19:48.555Z 13 Acked 1012 Flush 0 Done Done Sent false
98802024-02-06T22:19:48.555Z 14 Acked 1013 Write 10 Done Done Sent false
98812024-02-06T22:19:48.555Z 15 Acked 1014 Read 4 Done Done Sent false
98822024-02-06T22:19:48.555Z 16 Acked 1015 Write 3 Done Done Sent false
98832024-02-06T22:19:48.555Z 17 Acked 1016 Write 9 Done Done Sent false
98842024-02-06T22:19:48.555Z 18 Acked 1017 Read 6 Done Done Sent false
98852024-02-06T22:19:48.555Z 19 Acked 1018 Read 4 Done Done Sent false
98862024-02-06T22:19:48.555Z 20 Acked 1019 Write 10 Done Done Sent false
98872024-02-06T22:19:48.555Z 21 Acked 1020 Write 6 Done Done Sent false
98882024-02-06T22:19:48.555Z 22 Acked 1021 Read 1 Done Done Sent false
98892024-02-06T22:19:48.556Z 23 Acked 1022 Write 9 Done Done Sent false
98902024-02-06T22:19:48.556Z 24 Acked 1023 Write 1 Done Done Sent false
98912024-02-06T22:19:48.556Z 25 Acked 1024 Read 4 Done Done Sent false
98922024-02-06T22:19:48.556Z 26 Acked 1025 Flush 0 Done Done Sent false
98932024-02-06T22:19:48.556Z 27 Acked 1026 Write 3 Done Done Sent false
98942024-02-06T22:19:48.556Z 28 Acked 1027 Read 2 Done Done Sent false
98952024-02-06T22:19:48.556Z 29 Acked 1028 Read 3 Done Done Sent false
98962024-02-06T22:19:48.556Z 30 Acked 1029 Read 9 Done Sent Sent false
98972024-02-06T22:19:48.556Z STATES DS:0 DS:1 DS:2 TOTAL
98982024-02-06T22:19:48.556Z New 0 0 0 0
98992024-02-06T22:19:48.556Z Sent 0 1 27 28
99002024-02-06T22:19:48.556Z Done 30 29 3 62
99012024-02-06T22:19:48.556Z Skipped 0 0 0 0
99022024-02-06T22:19:48.556Z Error 0 0 0 0
99032024-02-06T22:19:48.556ZLast Flush: 1025 1025 0
99042024-02-06T22:19:48.556ZDownstairs last five completed:
99052024-02-06T22:19:48.570ZUpstairs last five completed: 30 29 28 27 26
99062024-02-06T22:19:48.570Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:47.734348508Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1498}
99072024-02-06T22:19:48.571ZWrote out file "/tmp/test_repair_verify.data"
99082024-02-06T22:19:48.571Z
99092024-02-06T22:19:48.571ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
99102024-02-06T22:19:48.571ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99112024-02-06T22:19:48.571Z 0 000-019 88 88 88 166 166 166 F F F
99122024-02-06T22:19:48.571Z 1 020-039 90 90 87 170 170 162 F F F <---
99132024-02-06T22:19:48.571Z 2 040-059 90 90 87 168 168 162 F F F <---
99142024-02-06T22:19:48.571Z 3 060-079 90 90 88 168 168 166 F F T <---
99152024-02-06T22:19:48.571Z 4 080-099 86 86 86 159 159 159 F F F
99162024-02-06T22:19:48.571Z 5 100-119 88 88 88 166 166 166 F F F
99172024-02-06T22:19:48.571Z 6 120-139 88 88 88 166 166 166 F F F
99182024-02-06T22:19:48.571Z 7 140-159 88 88 88 164 164 164 F F F
99192024-02-06T22:19:48.571Z 8 160-179 90 90 88 169 169 164 F F F <---
99202024-02-06T22:19:48.571Z 9 180-199 90 90 88 170 170 164 F F F <---
99212024-02-06T22:19:48.571Z 10 200-219 90 90 88 169 169 164 F F F <---
99222024-02-06T22:19:48.571Z 11 220-239 89 89 89 167 167 167 F F F
99232024-02-06T22:19:48.571Z 12 240-259 82 82 82 153 153 153 F F F
99242024-02-06T22:19:48.571Z 13 260-279 90 90 86 170 170 160 F F F <---
99252024-02-06T22:19:48.571Z 14 280-299 90 90 87 170 170 162 F F F <---
99262024-02-06T22:19:48.571Z 15 300-319 90 90 85 170 170 158 F F F <---
99272024-02-06T22:19:48.572Z 16 320-339 83 83 83 155 155 155 F F F
99282024-02-06T22:19:48.572Z 17 340-359 86 86 86 159 159 159 F F F
99292024-02-06T22:19:48.572Z 18 360-379 88 88 88 163 163 163 F F F
99302024-02-06T22:19:48.572Z 19 380-399 89 89 89 167 167 167 F F F
99312024-02-06T22:19:48.572Z 20 400-419 90 90 82 168 168 154 F F F <---
99322024-02-06T22:19:48.572Z 21 420-439 82 82 82 154 154 154 F F F
99332024-02-06T22:19:48.572Z 22 440-459 81 81 81 151 151 151 F F F
99342024-02-06T22:19:48.572Z 23 460-479 88 88 88 166 166 166 F F F
99352024-02-06T22:19:48.572Z 24 480-499 90 90 88 168 168 166 T T T <---
99362024-02-06T22:19:48.572Z 25 500-519 86 86 86 161 161 161 F F F
99372024-02-06T22:19:48.573Z 26 520-539 79 79 79 148 148 148 F F F
99382024-02-06T22:19:48.573Z 27 540-559 90 90 89 170 170 167 F F F <---
99392024-02-06T22:19:48.573Z 28 560-579 87 87 87 162 162 162 F F F
99402024-02-06T22:19:48.573Z 29 580-599 90 90 87 170 170 162 F F F <---
99412024-02-06T22:19:48.573ZMax gen: 90, Max flush: 170
99422024-02-06T22:19:48.573ZError: Difference in extent metadata found!
99432024-02-06T22:19:48.573ZA Difference in extent metadata is expected here
99442024-02-06T22:19:48.573ZOn loop 45
99452024-02-06T22:19:48.573Z
99462024-02-06T22:19:48.573ZVerifying data now
99472024-02-06T22:19:54.191ZNo change after verify
99482024-02-06T22:19:54.191ZLoop: 45 Downstairs dump after verify (and repair):
99492024-02-06T22:19:54.209ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99502024-02-06T22:19:54.209Z 0 000-019 88 88 88 166 166 166 F F F
99512024-02-06T22:19:54.209Z 1 020-039 90 90 90 170 170 170 F F F
99522024-02-06T22:19:54.209Z 2 040-059 90 90 90 168 168 168 F F F
99532024-02-06T22:19:54.209Z 3 060-079 90 90 90 168 168 168 F F F
99542024-02-06T22:19:54.209Z 4 080-099 86 86 86 159 159 159 F F F
99552024-02-06T22:19:54.209Z 5 100-119 88 88 88 166 166 166 F F F
99562024-02-06T22:19:54.209Z 6 120-139 88 88 88 166 166 166 F F F
99572024-02-06T22:19:54.209Z 7 140-159 88 88 88 164 164 164 F F F
99582024-02-06T22:19:54.209Z 8 160-179 90 90 90 169 169 169 F F F
99592024-02-06T22:19:54.210Z 9 180-199 90 90 90 170 170 170 F F F
99602024-02-06T22:19:54.210Z 10 200-219 90 90 90 169 169 169 F F F
99612024-02-06T22:19:54.210Z 11 220-239 89 89 89 167 167 167 F F F
99622024-02-06T22:19:54.210Z 12 240-259 82 82 82 153 153 153 F F F
99632024-02-06T22:19:54.210Z 13 260-279 90 90 90 170 170 170 F F F
99642024-02-06T22:19:54.210Z 14 280-299 90 90 90 170 170 170 F F F
99652024-02-06T22:19:54.210Z 15 300-319 90 90 90 170 170 170 F F F
99662024-02-06T22:19:54.210Z 16 320-339 83 83 83 155 155 155 F F F
99672024-02-06T22:19:54.210Z 17 340-359 86 86 86 159 159 159 F F F
99682024-02-06T22:19:54.210Z 18 360-379 88 88 88 163 163 163 F F F
99692024-02-06T22:19:54.210Z 19 380-399 89 89 89 167 167 167 F F F
99702024-02-06T22:19:54.210Z 20 400-419 90 90 90 168 168 168 F F F
99712024-02-06T22:19:54.210Z 21 420-439 82 82 82 154 154 154 F F F
99722024-02-06T22:19:54.211Z 22 440-459 81 81 81 151 151 151 F F F
99732024-02-06T22:19:54.211Z 23 460-479 88 88 88 166 166 166 F F F
99742024-02-06T22:19:54.211Z 24 480-499 91 91 91 171 171 171 F F F
99752024-02-06T22:19:54.211Z 25 500-519 86 86 86 161 161 161 F F F
99762024-02-06T22:19:54.211Z 26 520-539 79 79 79 148 148 148 F F F
99772024-02-06T22:19:54.211Z 27 540-559 90 90 90 170 170 170 F F F
99782024-02-06T22:19:54.211Z 28 560-579 87 87 87 162 162 162 F F F
99792024-02-06T22:19:54.211Z 29 580-599 90 90 90 170 170 170 F F F
99802024-02-06T22:19:54.211ZMax gen: 91, Max flush: 171
99812024-02-06T22:19:54.212Z
99822024-02-06T22:19:54.212ZBegin loop 46
99832024-02-06T22:19:54.212ZDownstairs to restart: 1
99842024-02-06T22:19:54.239Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.431639485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506}
99852024-02-06T22:19:54.239Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.431915247Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506}
99862024-02-06T22:19:54.239Z{"msg":"Crucible runtime is spawned
99872024-02-06T22:19:54.239ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.43196956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506}
99882024-02-06T22:19:54.239Z{"msg":"Crucible 79a5577a-77d5-4d2d-9186-1cfb383f1f49 has session id: a136980e-7d8d-458f-85ef-f45e243bac00","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432016184Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99892024-02-06T22:19:54.239Z{"msg":"Upstairs opts: Upstairs UUID: 79a5577a-77d5-4d2d-9186-1cfb383f1f49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432055419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99902024-02-06T22:19:54.239Z{"msg":"Crucible stats registered with UUID: 79a5577a-77d5-4d2d-9186-1cfb383f1f49","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432093763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99912024-02-06T22:19:54.239Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432111481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506}
99922024-02-06T22:19:54.239Z{"msg":"79a5577a-77d5-4d2d-9186-1cfb383f1f49 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432170803Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99932024-02-06T22:19:54.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432218146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99942024-02-06T22:19:54.239Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.43227063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99952024-02-06T22:19:54.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432303535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99962024-02-06T22:19:54.239Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432332861Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99972024-02-06T22:19:54.239Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432362427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99982024-02-06T22:19:54.239Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432394653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
99992024-02-06T22:19:54.239Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432436127Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100002024-02-06T22:19:54.239Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.43248913Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100012024-02-06T22:19:54.239Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432549762Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100022024-02-06T22:19:54.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432613413Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100032024-02-06T22:19:54.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432671735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100042024-02-06T22:19:54.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.432699672Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100052024-02-06T22:19:54.240Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.433134313Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100062024-02-06T22:19:54.240Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.433163529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100072024-02-06T22:19:54.240Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.433199714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100082024-02-06T22:19:54.240Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.43322574Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100092024-02-06T22:19:54.240Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.433797883Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100102024-02-06T22:19:54.240Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.433954462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100112024-02-06T22:19:54.240Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434290166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100122024-02-06T22:19:54.240Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434322112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100132024-02-06T22:19:54.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434437297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100142024-02-06T22:19:54.240Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434472032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100152024-02-06T22:19:54.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.4344878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100162024-02-06T22:19:54.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434530614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100172024-02-06T22:19:54.240Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434545792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100182024-02-06T22:19:54.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434566479Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100192024-02-06T22:19:54.240Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.434850321Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100202024-02-06T22:19:54.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435261875Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100212024-02-06T22:19:54.240Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435293991Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100222024-02-06T22:19:54.240Z{"msg":"[0]R flush_numbers[0..12]: [166, 170, 168, 168, 159, 166, 166, 164, 169, 170, 169, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435320907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100232024-02-06T22:19:54.240Z{"msg":"[0]R generation[0..12]: [88, 90, 90, 90, 86, 88, 88, 88, 90, 90, 90, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435351363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100242024-02-06T22:19:54.240Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435377999Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100252024-02-06T22:19:54.240Z{"msg":"[1]R flush_numbers[0..12]: [166, 170, 168, 168, 159, 166, 166, 164, 169, 170, 169, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435406676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100262024-02-06T22:19:54.240Z{"msg":"[1]R generation[0..12]: [88, 90, 90, 90, 86, 88, 88, 88, 90, 90, 90, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435431352Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100272024-02-06T22:19:54.240ZWait for a query_work_queue command to finish before sending IO
100282024-02-06T22:19:54.241Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435464548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100292024-02-06T22:19:54.241Z{"msg":"[2]R flush_numbers[0..12]: [166, 170, 168, 168, 159, 166, 166, 164, 169, 170, 169, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435491264Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100302024-02-06T22:19:54.241Z{"msg":"[2]R generation[0..12]: [88, 90, 90, 90, 86, 88, 88, 88, 90, 90, 90, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.43552128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100312024-02-06T22:19:54.241Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435547487Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100322024-02-06T22:19:54.241Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435574813Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100332024-02-06T22:19:54.241Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.43559855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100342024-02-06T22:19:54.241Z{"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435625396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100352024-02-06T22:19:54.241Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435649253Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100362024-02-06T22:19:54.241Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435680419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100372024-02-06T22:19:54.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435699276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100382024-02-06T22:19:54.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435713314Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100392024-02-06T22:19:54.241Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435740411Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100402024-02-06T22:19:54.241Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435760118Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100412024-02-06T22:19:54.241Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435791394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100422024-02-06T22:19:54.241Z{"msg":"79a5577a-77d5-4d2d-9186-1cfb383f1f49 is now active with session: a136980e-7d8d-458f-85ef-f45e243bac00","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435806402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100432024-02-06T22:19:54.241Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435835968Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506,"session_id":"a136980e-7d8d-458f-85ef-f45e243bac00"}
100442024-02-06T22:19:54.241Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:53.435856625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506}
100452024-02-06T22:19:54.241ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
100462024-02-06T22:19:54.241ZLoading write count information from file "/tmp/test_repair_verify.data"
100472024-02-06T22:19:54.241ZRun Repair workload
100482024-02-06T22:19:54.241Z01/30 Write block 454 len 8192 data: 3 3
100492024-02-06T22:19:54.241Z02/30 Read block 188 len 12288
100502024-02-06T22:19:54.241Z03/30 Write block 353 len 12288 data: 12 11 14
100512024-02-06T22:19:54.242Z04/30 Read block 191 len 40960
100522024-02-06T22:19:54.242Z05/30 Write block 120 len 40960 data: 8 9 10 10 10 11 11 12 13 11
100532024-02-06T22:19:54.242Z06/30 Write block 93 len 20480 data: 4 5 5 5 5
100542024-02-06T22:19:54.242Z07/30 Write block 105 len 4096 data: 5
100552024-02-06T22:19:54.251Z08/30 Write block 215 len 40960 data: 8 6 6 7 7 12 12 11 11 11
100562024-02-06T22:19:54.252Z09/30 Read block 445 len 20480
100572024-02-06T22:19:54.252Z10/30 Read block 344 len 4096
100582024-02-06T22:19:54.252Z11/30 Read block 584 len 24576
100592024-02-06T22:19:54.252Z12/30 Write block 229 len 16384 data: 7 7 10 10
100602024-02-06T22:19:54.252Z13/30 Write block 247 len 12288 data: 10 10 11
100612024-02-06T22:19:54.252Z14/30 Write block 437 len 36864 data: 10 9 8 6 4 6 6 5 5
100622024-02-06T22:19:54.252Z15/30 Write block 315 len 24576 data: 10 9 7 7 8 6
100632024-02-06T22:19:54.252Z16/30 Flush
100642024-02-06T22:19:54.252Z17/30 Write block 81 len 12288 data: 5 7 7
100652024-02-06T22:19:54.252Z18/30 Write block 535 len 4096 data: 6
100662024-02-06T22:19:54.252Z19/30 Read block 275 len 36864
100672024-02-06T22:19:54.252Z20/30 Read block 18 len 32768
100682024-02-06T22:19:54.252Z21/30 Write block 234 len 16384 data: 11 10 9 8
100692024-02-06T22:19:54.252Z22/30 Flush
100702024-02-06T22:19:54.252Z23/30 Write block 315 len 16384 data: 11 10 8 8
100712024-02-06T22:19:54.252Z24/30 Write block 21 len 20480 data: 5 7 7 7 8
100722024-02-06T22:19:54.271Z25/30 Flush
100732024-02-06T22:19:54.271Z26/30 Write block 299 len 20480 data: 6 8 9 8 8
100742024-02-06T22:19:54.271Z27/30 Read block 581 len 8192
100752024-02-06T22:19:54.271Z28/30 Write block 374 len 32768 data: 7 7 9 12 11 9 9 10
100762024-02-06T22:19:54.271Z29/30 Read block 241 len 4096
100772024-02-06T22:19:54.271Z30/30 Write block 566 len 40960 data: 8 10 10 8 8 11 12 11 12 11
100782024-02-06T22:19:54.271Z----------------------------------------------------------------
100792024-02-06T22:19:54.271Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
100802024-02-06T22:19:54.272ZStates: Active Active Active
100812024-02-06T22:19:54.272ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
100822024-02-06T22:19:54.272Z 1 Acked 1000 Write 2 Done Done Done false
100832024-02-06T22:19:54.272Z 2 Acked 1001 Read 3 Done Done Done false
100842024-02-06T22:19:54.272Z 3 Acked 1002 Write 3 Done Done Done false
100852024-02-06T22:19:54.272Z 4 Acked 1003 Read 10 Done Done Done false
100862024-02-06T22:19:54.272Z 5 Acked 1004 Write 10 Done Sent Done false
100872024-02-06T22:19:54.272Z 6 Acked 1005 Write 5 Done Sent Done false
100882024-02-06T22:19:54.272Z 7 Acked 1006 Write 1 Done Sent Done false
100892024-02-06T22:19:54.272Z 8 Acked 1007 Write 10 Done Sent Done false
100902024-02-06T22:19:54.272Z 9 Acked 1008 Read 5 Done Sent Done false
100912024-02-06T22:19:54.272Z 10 Acked 1009 Read 1 Done Sent Done false
100922024-02-06T22:19:54.272Z 11 Acked 1010 Read 6 Done Sent Done false
100932024-02-06T22:19:54.272Z 12 Acked 1011 Write 4 Done Sent Done false
100942024-02-06T22:19:54.272Z 13 Acked 1012 Write 3 Done Sent Done false
100952024-02-06T22:19:54.272Z 14 Acked 1013 Write 9 Done Sent Done false
100962024-02-06T22:19:54.272Z 15 Acked 1014 Write 6 Done Sent Done false
100972024-02-06T22:19:54.272Z 16 Acked 1015 Flush 0 Done Sent Done false
100982024-02-06T22:19:54.272Z 17 Acked 1016 Write 3 Done Sent Done false
100992024-02-06T22:19:54.272Z 18 Acked 1017 Write 1 Done Sent Done false
101002024-02-06T22:19:54.273Z 19 Acked 1018 Read 9 Done Sent Done false
101012024-02-06T22:19:54.273Z 20 Acked 1019 Read 8 Done Sent Done false
101022024-02-06T22:19:54.273Z 21 Acked 1020 Write 4 Done Sent Done false
101032024-02-06T22:19:54.273Z 22 Acked 1021 Flush 0 Done Sent Done false
101042024-02-06T22:19:54.273Z 23 Acked 1022 Write 4 Done Sent Done false
101052024-02-06T22:19:54.273Z 24 Acked 1023 Write 5 Done Sent Done false
101062024-02-06T22:19:54.273Z 25 Acked 1024 Flush 0 Done Sent Done false
101072024-02-06T22:19:54.273Z 26 Acked 1025 Write 5 Done Sent Done false
101082024-02-06T22:19:54.273Z 27 Acked 1026 Read 2 Done Sent Done false
101092024-02-06T22:19:54.273Z 28 Acked 1027 Write 8 Done Sent Done false
101102024-02-06T22:19:54.273Z 29 Acked 1028 Read 1 Done Sent Done false
101112024-02-06T22:19:54.273Z 30 Acked 1029 Write 10 Sent Sent Sent false
101122024-02-06T22:19:54.273Z STATES DS:0 DS:1 DS:2 TOTAL
101132024-02-06T22:19:54.273Z New 0 0 0 0
101142024-02-06T22:19:54.273Z Sent 1 26 1 28
101152024-02-06T22:19:54.273Z Done 29 4 29 62
101162024-02-06T22:19:54.274Z Skipped 0 0 0 0
101172024-02-06T22:19:54.274Z Error 0 0 0 0
101182024-02-06T22:19:54.274ZLast Flush: 1024 0 1024
101192024-02-06T22:19:54.274ZDownstairs last five completed:
101202024-02-06T22:19:54.274ZUpstairs last five completed: 30 29 28 27 26
101212024-02-06T22:19:54.274Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:53.466702927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1506}
101222024-02-06T22:19:54.274ZWrote out file "/tmp/test_repair_verify.data"
101232024-02-06T22:19:54.274Z
101242024-02-06T22:19:54.290ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
101252024-02-06T22:19:54.290ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101262024-02-06T22:19:54.290Z 0 000-019 88 88 88 166 166 166 F F F
101272024-02-06T22:19:54.290Z 1 020-039 92 90 92 174 170 174 F F F <---
101282024-02-06T22:19:54.290Z 2 040-059 90 90 90 168 168 168 F F F
101292024-02-06T22:19:54.290Z 3 060-079 90 90 90 168 168 168 F F F
101302024-02-06T22:19:54.290Z 4 080-099 92 86 92 173 159 173 F F F <---
101312024-02-06T22:19:54.290Z 5 100-119 92 88 92 172 166 172 F F F <---
101322024-02-06T22:19:54.290Z 6 120-139 92 88 92 172 166 172 F F F <---
101332024-02-06T22:19:54.290Z 7 140-159 88 88 88 164 164 164 F F F
101342024-02-06T22:19:54.290Z 8 160-179 90 90 90 169 169 169 F F F
101352024-02-06T22:19:54.290Z 9 180-199 90 90 90 170 170 170 F F F
101362024-02-06T22:19:54.290Z 10 200-219 92 90 92 172 169 172 F F F <---
101372024-02-06T22:19:54.290Z 11 220-239 92 89 92 173 167 173 F F F <---
101382024-02-06T22:19:54.290Z 12 240-259 92 82 92 172 153 172 F F F <---
101392024-02-06T22:19:54.290Z 13 260-279 90 90 90 170 170 170 F F F
101402024-02-06T22:19:54.290Z 14 280-299 90 90 90 170 170 170 T F T <---
101412024-02-06T22:19:54.290Z 15 300-319 92 90 92 174 170 174 T F T <---
101422024-02-06T22:19:54.291Z 16 320-339 92 83 92 172 155 172 F F F <---
101432024-02-06T22:19:54.291Z 17 340-359 92 86 92 172 159 172 F T F <---
101442024-02-06T22:19:54.291Z 18 360-379 88 88 88 163 163 163 T F T <---
101452024-02-06T22:19:54.291Z 19 380-399 89 89 89 167 167 167 T F T <---
101462024-02-06T22:19:54.291Z 20 400-419 90 90 90 168 168 168 F F F
101472024-02-06T22:19:54.291Z 21 420-439 92 82 92 172 154 172 F F F <---
101482024-02-06T22:19:54.291Z 22 440-459 92 81 92 172 151 172 F T F <---
101492024-02-06T22:19:54.291Z 23 460-479 88 88 88 166 166 166 F F F
101502024-02-06T22:19:54.291Z 24 480-499 91 91 91 171 171 171 F F F
101512024-02-06T22:19:54.291Z 25 500-519 86 86 86 161 161 161 F F F
101522024-02-06T22:19:54.291Z 26 520-539 92 79 92 173 148 173 F F F <---
101532024-02-06T22:19:54.291Z 27 540-559 90 90 90 170 170 170 F F F
101542024-02-06T22:19:54.291Z 28 560-579 87 87 87 162 162 162 T F T <---
101552024-02-06T22:19:54.291Z 29 580-599 90 90 90 170 170 170 F F F
101562024-02-06T22:19:54.291ZMax gen: 92, Max flush: 174
101572024-02-06T22:19:54.291ZError: Difference in extent metadata found!
101582024-02-06T22:19:54.291ZA Difference in extent metadata is expected here
101592024-02-06T22:19:54.291ZOn loop 46
101602024-02-06T22:19:54.291Z
101612024-02-06T22:19:54.291ZVerifying data now
101622024-02-06T22:20:00.102ZNo change after verify
101632024-02-06T22:20:00.102ZLoop: 46 Downstairs dump after verify (and repair):
101642024-02-06T22:20:00.117ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101652024-02-06T22:20:00.117Z 0 000-019 88 88 88 166 166 166 F F F
101662024-02-06T22:20:00.117Z 1 020-039 92 92 92 174 174 174 F F F
101672024-02-06T22:20:00.117Z 2 040-059 90 90 90 168 168 168 F F F
101682024-02-06T22:20:00.117Z 3 060-079 90 90 90 168 168 168 F F F
101692024-02-06T22:20:00.117Z 4 080-099 92 92 92 173 173 173 F F F
101702024-02-06T22:20:00.145Z 5 100-119 92 92 92 172 172 172 F F F
101712024-02-06T22:20:00.145Z 6 120-139 92 92 92 172 172 172 F F F
101722024-02-06T22:20:00.145Z 7 140-159 88 88 88 164 164 164 F F F
101732024-02-06T22:20:00.145Z 8 160-179 90 90 90 169 169 169 F F F
101742024-02-06T22:20:00.145Z 9 180-199 90 90 90 170 170 170 F F F
101752024-02-06T22:20:00.145Z 10 200-219 92 92 92 172 172 172 F F F
101762024-02-06T22:20:00.145Z 11 220-239 92 92 92 173 173 173 F F F
101772024-02-06T22:20:00.145Z 12 240-259 92 92 92 172 172 172 F F F
101782024-02-06T22:20:00.145Z 13 260-279 90 90 90 170 170 170 F F F
101792024-02-06T22:20:00.145Z 14 280-299 93 93 93 175 175 175 F F F
101802024-02-06T22:20:00.145Z 15 300-319 93 93 93 175 175 175 F F F
101812024-02-06T22:20:00.145Z 16 320-339 92 92 92 172 172 172 F F F
101822024-02-06T22:20:00.145Z 17 340-359 92 92 92 172 172 172 F F F
101832024-02-06T22:20:00.145Z 18 360-379 93 93 93 175 175 175 F F F
101842024-02-06T22:20:00.145Z 19 380-399 93 93 93 175 175 175 F F F
101852024-02-06T22:20:00.146Z 20 400-419 90 90 90 168 168 168 F F F
101862024-02-06T22:20:00.146Z 21 420-439 92 92 92 172 172 172 F F F
101872024-02-06T22:20:00.146Z 22 440-459 92 92 92 172 172 172 F F F
101882024-02-06T22:20:00.146Z 23 460-479 88 88 88 166 166 166 F F F
101892024-02-06T22:20:00.146Z 24 480-499 91 91 91 171 171 171 F F F
101902024-02-06T22:20:00.146Z 25 500-519 86 86 86 161 161 161 F F F
101912024-02-06T22:20:00.146Z 26 520-539 92 92 92 173 173 173 F F F
101922024-02-06T22:20:00.146Z 27 540-559 90 90 90 170 170 170 F F F
101932024-02-06T22:20:00.146Z 28 560-579 93 93 93 175 175 175 F F F
101942024-02-06T22:20:00.146Z 29 580-599 90 90 90 170 170 170 F F F
101952024-02-06T22:20:00.146ZMax gen: 93, Max flush: 175
101962024-02-06T22:20:00.147Z
101972024-02-06T22:20:00.147ZBegin loop 47
101982024-02-06T22:20:00.147ZDownstairs to restart: 1
101992024-02-06T22:20:00.147Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345198351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514}
102002024-02-06T22:20:00.147Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
102012024-02-06T22:20:00.147Z rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345526007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514}
102022024-02-06T22:20:00.147Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345562432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514}
102032024-02-06T22:20:00.147Z{"msg":"Crucible fcb5460f-9f4f-47d3-b675-140bb183fd57 has session id: c386f216-37f3-48be-8976-ae648e0ee3c8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345600377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102042024-02-06T22:20:00.147Z{"msg":"Upstairs opts: Upstairs UUID: fcb5460f-9f4f-47d3-b675-140bb183fd57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345636712Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102052024-02-06T22:20:00.147Z{"msg":"Crucible stats registered with UUID: fcb5460f-9f4f-47d3-b675-140bb183fd57","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345683106Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102062024-02-06T22:20:00.147Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345704623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514}
102072024-02-06T22:20:00.147Z{"msg":"fcb5460f-9f4f-47d3-b675-140bb183fd57 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.34572793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102082024-02-06T22:20:00.147Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345758686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102092024-02-06T22:20:00.147Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345777543Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102102024-02-06T22:20:00.147Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.34580022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102112024-02-06T22:20:00.147Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345841604Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102122024-02-06T22:20:00.147Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345881479Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102132024-02-06T22:20:00.147Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.345963928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102142024-02-06T22:20:00.147Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346031809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102152024-02-06T22:20:00.147Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.34609401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102162024-02-06T22:20:00.147Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346121997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102172024-02-06T22:20:00.148Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346147163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102182024-02-06T22:20:00.148Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346174309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102192024-02-06T22:20:00.148Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346205975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102202024-02-06T22:20:00.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346847039Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102212024-02-06T22:20:00.148Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346922338Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102222024-02-06T22:20:00.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.346949765Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102232024-02-06T22:20:00.148Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.34698682Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102242024-02-06T22:20:00.148Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.347460656Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102252024-02-06T22:20:00.148Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.347610425Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102262024-02-06T22:20:00.148Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.347979166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102272024-02-06T22:20:00.148Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.348028679Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102282024-02-06T22:20:00.148Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.348057235Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102292024-02-06T22:20:00.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.348108068Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102302024-02-06T22:20:00.148Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.348151772Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102312024-02-06T22:20:00.148Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.348195816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102322024-02-06T22:20:00.148Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.348227022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102332024-02-06T22:20:00.148Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.34824213Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102342024-02-06T22:20:00.148Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.348702578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102352024-02-06T22:20:00.148Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349162396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102362024-02-06T22:20:00.148Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349238225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102372024-02-06T22:20:00.148Z{"msg":"[0]R flush_numbers[0..12]: [166, 174, 168, 168, 173, 172, 172, 164, 169, 170, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349257433Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102382024-02-06T22:20:00.148Z{"msg":"[0]R generation[0..12]: [88, 92, 90, 90, 92, 92, 92, 88, 90, 90, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349284979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102392024-02-06T22:20:00.148Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349310236Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102402024-02-06T22:20:00.148Z{"msg":"[1]R flush_numbers[0..12]: [166, 174, 168, 168, 173, 172, 172, 164, 169, 170, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349339212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102412024-02-06T22:20:00.156Z{"msg":"[1]R generation[0..12]: [88, 92, 90, 90, 92, 92, 92, 88, 90, 90, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349355329Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102422024-02-06T22:20:00.156Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
102432024-02-06T22:20:00.156Z,"time":"2024-02-06T22:19:59.349391495Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102442024-02-06T22:20:00.156Z{"msg":"[2]R flush_numbers[0..12]: [166, 174, 168, 168, 173, 172, 172, 164, 169, 170, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349422231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102452024-02-06T22:20:00.156Z{"msg":"[2]R generation[0..12]: [88, 92, 90, 90, 92, 92, 92, 88, 90, 90, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349452607Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102462024-02-06T22:20:00.156Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349468974Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102472024-02-06T22:20:00.156Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349482522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102482024-02-06T22:20:00.157Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349508589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102492024-02-06T22:20:00.157Z{"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349533505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102502024-02-06T22:20:00.157Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349564951Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102512024-02-06T22:20:00.157Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349588948Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102522024-02-06T22:20:00.157Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349623933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102532024-02-06T22:20:00.157Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.34965085Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102542024-02-06T22:20:00.157Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349678696Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102552024-02-06T22:20:00.157Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349693954Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102562024-02-06T22:20:00.157Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.34972504Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102572024-02-06T22:20:00.157Z{"msg":"fcb5460f-9f4f-47d3-b675-140bb183fd57 is now active with session: c386f216-37f3-48be-8976-ae648e0ee3c8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349751926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102582024-02-06T22:20:00.157Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349779162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514,"session_id":"c386f216-37f3-48be-8976-ae648e0ee3c8"}
102592024-02-06T22:20:00.158Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:19:59.349803149Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514}
102602024-02-06T22:20:00.158ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
102612024-02-06T22:20:00.158ZLoading write count information from file "/tmp/test_repair_verify.data"
102622024-02-06T22:20:00.158ZRun Repair workload
102632024-02-06T22:20:00.158Z01/30 Write block 257 len 32768 data: 5 4 5 7 10 10 9 8
102642024-02-06T22:20:00.158Z02/30 Read block 127 len 24576
102652024-02-06T22:20:00.158Z03/30 Write block 235 len 20480 data: 11 10 9 6 5
102662024-02-06T22:20:00.158Z04/30 Write block 288 len 36864 data: 9 10 13 13 10 11 11 12 9
102672024-02-06T22:20:00.158Z05/30 Flush
102682024-02-06T22:20:00.158Z06/30 Write block 344 len 4096 data: 6
102692024-02-06T22:20:00.158Z07/30 Read block 185 len 8192
102702024-02-06T22:20:00.158Z08/30 Write block 214 len 36864 data: 9 9 7 7 8 8 13 13 12
102712024-02-06T22:20:00.158Z09/30 Read block 397 len 36864
102722024-02-06T22:20:00.158Z10/30 Flush
102732024-02-06T22:20:00.162Z11/30 Write block 296 len 36864 data: 10 7 6 7 9 10 9 9 7
102742024-02-06T22:20:00.163Z12/30 Flush
102752024-02-06T22:20:00.163Z13/30 Write block 197 len 40960 data: 10 7 7 6 7 10 9 9 8 10
102762024-02-06T22:20:00.163Z14/30 Read block 294 len 36864
102772024-02-06T22:20:00.163Z15/30 Read block 496 len 32768
102782024-02-06T22:20:00.179Z16/30 Read block 237 len 40960
102792024-02-06T22:20:00.179Z17/30 Read block 234 len 36864
102802024-02-06T22:20:00.179Z18/30 Read block 434 len 20480
102812024-02-06T22:20:00.179Z19/30 Write block 16 len 12288 data: 6 5 5
102822024-02-06T22:20:00.179Z20/30 Read block 39 len 32768
102832024-02-06T22:20:00.179Z21/30 Read block 212 len 20480
102842024-02-06T22:20:00.179Z22/30 Write block 322 len 8192 data: 6 5
102852024-02-06T22:20:00.179Z23/30 Read block 405 len 36864
102862024-02-06T22:20:00.179Z24/30 Read block 163 len 20480
102872024-02-06T22:20:00.179Z25/30 Write block 455 len 12288 data: 4 3 4
102882024-02-06T22:20:00.179Z26/30 Write block 486 len 12288 data: 7 7 6
102892024-02-06T22:20:00.179Z27/30 Read block 219 len 24576
102902024-02-06T22:20:00.179Z28/30 Read block 185 len 16384
102912024-02-06T22:20:00.179Z29/30 Read block 416 len 12288
102922024-02-06T22:20:00.179Z30/30 Write block 173 len 12288 data: 12 12 11
102932024-02-06T22:20:00.180Z----------------------------------------------------------------
102942024-02-06T22:20:00.180Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:25
102952024-02-06T22:20:00.180ZStates: Active Active Active
102962024-02-06T22:20:00.180ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102972024-02-06T22:20:00.180Z 6 Acked 1005 Write 1 Done Done Done false
102982024-02-06T22:20:00.180Z 7 Acked 1006 Read 2 Done Done Done false
102992024-02-06T22:20:00.180Z 8 Acked 1007 Write 9 Done Sent Done false
103002024-02-06T22:20:00.180Z 9 Acked 1008 Read 9 Done Sent Done false
103012024-02-06T22:20:00.180Z 10 Acked 1009 Flush 0 Done Sent Done false
103022024-02-06T22:20:00.180Z 11 Acked 1010 Write 9 Done Sent Done false
103032024-02-06T22:20:00.180Z 12 Acked 1011 Flush 0 Done Sent Done false
103042024-02-06T22:20:00.180Z 13 Acked 1012 Write 10 Done Sent Done false
103052024-02-06T22:20:00.180Z 14 Acked 1013 Read 9 Done Sent Done false
103062024-02-06T22:20:00.180Z 15 Acked 1014 Read 8 Done Sent Done false
103072024-02-06T22:20:00.180Z 16 Acked 1015 Read 10 Done Sent Done false
103082024-02-06T22:20:00.180Z 17 Acked 1016 Read 9 Done Sent Done false
103092024-02-06T22:20:00.180Z 18 Acked 1017 Read 5 Done Sent Done false
103102024-02-06T22:20:00.180Z 19 Acked 1018 Write 3 Done Sent Done false
103112024-02-06T22:20:00.180Z 20 Acked 1019 Read 8 Done Sent Done false
103122024-02-06T22:20:00.180Z 21 Acked 1020 Read 5 Done Sent Done false
103132024-02-06T22:20:00.180Z 22 Acked 1021 Write 2 Done Sent Done false
103142024-02-06T22:20:00.180Z 23 Acked 1022 Read 9 Done Sent Done false
103152024-02-06T22:20:00.180Z 24 Acked 1023 Read 5 Done Sent Done false
103162024-02-06T22:20:00.180Z 25 Acked 1024 Write 3 Done Sent Done false
103172024-02-06T22:20:00.181Z 26 Acked 1025 Write 3 Done Sent Done false
103182024-02-06T22:20:00.181Z 27 Acked 1026 Read 6 Done Sent Done false
103192024-02-06T22:20:00.181Z 28 Acked 1027 Read 4 Done Sent Done false
103202024-02-06T22:20:00.181Z 29 Acked 1028 Read 3 Done Sent Done false
103212024-02-06T22:20:00.181Z 30 Acked 1029 Write 3 Sent Sent Sent false
103222024-02-06T22:20:00.181Z STATES DS:0 DS:1 DS:2 TOTAL
103232024-02-06T22:20:00.181Z New 0 0 0 0
103242024-02-06T22:20:00.181Z Sent 1 23 1 25
103252024-02-06T22:20:00.181Z Done 24 2 24 50
103262024-02-06T22:20:00.181Z Skipped 0 0 0 0
103272024-02-06T22:20:00.181Z Error 0 0 0 0
103282024-02-06T22:20:00.181ZLast Flush: 1011 1004 1011
103292024-02-06T22:20:00.204ZDownstairs last five completed: 1004 1003 1002 1001 1000
103302024-02-06T22:20:00.204ZUpstairs last five completed: 30 29 28 27 26
103312024-02-06T22:20:00.205Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:19:59.382911916Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1514}
103322024-02-06T22:20:00.205ZWrote out file "/tmp/test_repair_verify.data"
103332024-02-06T22:20:00.205Z
103342024-02-06T22:20:00.205ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
103352024-02-06T22:20:00.212ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103362024-02-06T22:20:00.212Z 0 000-019 88 88 88 166 166 166 T F T <---
103372024-02-06T22:20:00.212Z 1 020-039 92 92 92 174 174 174 F F F
103382024-02-06T22:20:00.212Z 2 040-059 90 90 90 168 168 168 F F F
103392024-02-06T22:20:00.212Z 3 060-079 90 90 90 168 168 168 F F F
103402024-02-06T22:20:00.212Z 4 080-099 92 92 92 173 173 173 F F F
103412024-02-06T22:20:00.212Z 5 100-119 92 92 92 172 172 172 F F F
103422024-02-06T22:20:00.212Z 6 120-139 92 92 92 172 172 172 F F F
103432024-02-06T22:20:00.212Z 7 140-159 88 88 88 164 164 164 F F F
103442024-02-06T22:20:00.212Z 8 160-179 90 90 90 169 169 169 T F T <---
103452024-02-06T22:20:00.212Z 9 180-199 90 90 90 170 170 170 T F T <---
103462024-02-06T22:20:00.212Z 10 200-219 94 92 94 177 172 177 T F T <---
103472024-02-06T22:20:00.212Z 11 220-239 94 94 94 177 176 177 F F F <---
103482024-02-06T22:20:00.212Z 12 240-259 94 94 94 176 176 176 F F F
103492024-02-06T22:20:00.212Z 13 260-279 94 94 94 176 176 176 F F F
103502024-02-06T22:20:00.212Z 14 280-299 94 94 94 178 176 178 F F F <---
103512024-02-06T22:20:00.212Z 15 300-319 94 93 94 178 175 178 F F F <---
103522024-02-06T22:20:00.212Z 16 320-339 92 92 92 172 172 172 T F T <---
103532024-02-06T22:20:00.213Z 17 340-359 94 92 94 177 172 177 F T F <---
103542024-02-06T22:20:00.213Z 18 360-379 93 93 93 175 175 175 F F F
103552024-02-06T22:20:00.213Z 19 380-399 93 93 93 175 175 175 F F F
103562024-02-06T22:20:00.213Z 20 400-419 90 90 90 168 168 168 F F F
103572024-02-06T22:20:00.213Z 21 420-439 92 92 92 172 172 172 F F F
103582024-02-06T22:20:00.213Z 22 440-459 92 92 92 172 172 172 T F T <---
103592024-02-06T22:20:00.213Z 23 460-479 88 88 88 166 166 166 F F F
103602024-02-06T22:20:00.213Z 24 480-499 91 91 91 171 171 171 T F T <---
103612024-02-06T22:20:00.213Z 25 500-519 86 86 86 161 161 161 F F F
103622024-02-06T22:20:00.213Z 26 520-539 92 92 92 173 173 173 F F F
103632024-02-06T22:20:00.213Z 27 540-559 90 90 90 170 170 170 F F F
103642024-02-06T22:20:00.213Z 28 560-579 93 93 93 175 175 175 F F F
103652024-02-06T22:20:00.213Z 29 580-599 90 90 90 170 170 170 F F F
103662024-02-06T22:20:00.213ZMax gen: 94, Max flush: 178
103672024-02-06T22:20:00.213ZError: Difference in extent metadata found!
103682024-02-06T22:20:00.213ZA Difference in extent metadata is expected here
103692024-02-06T22:20:00.213ZOn loop 47
103702024-02-06T22:20:00.214Z
103712024-02-06T22:20:00.214ZVerifying data now
103722024-02-06T22:20:08.560ZNo change after verify
103732024-02-06T22:20:08.560ZLoop: 47 Downstairs dump after verify (and repair):
103742024-02-06T22:20:08.600ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103752024-02-06T22:20:08.600Z 0 000-019 95 95 95 179 179 179 F F F
103762024-02-06T22:20:08.600Z 1 020-039 92 92 92 174 174 174 F F F
103772024-02-06T22:20:08.600Z 2 040-059 90 90 90 168 168 168 F F F
103782024-02-06T22:20:08.601Z 3 060-079 90 90 90 168 168 168 F F F
103792024-02-06T22:20:08.601Z 4 080-099 92 92 92 173 173 173 F F F
103802024-02-06T22:20:08.601Z 5 100-119 92 92 92 172 172 172 F F F
103812024-02-06T22:20:08.601Z 6 120-139 92 92 92 172 172 172 F F F
103822024-02-06T22:20:08.601Z 7 140-159 88 88 88 164 164 164 F F F
103832024-02-06T22:20:08.601Z 8 160-179 95 95 95 179 179 179 F F F
103842024-02-06T22:20:08.601Z 9 180-199 95 95 95 179 179 179 F F F
103852024-02-06T22:20:08.601Z 10 200-219 95 95 95 179 179 179 F F F
103862024-02-06T22:20:08.602Z 11 220-239 94 94 94 177 177 177 F F F
103872024-02-06T22:20:08.602Z 12 240-259 94 94 94 176 176 176 F F F
103882024-02-06T22:20:08.602Z 13 260-279 94 94 94 176 176 176 F F F
103892024-02-06T22:20:08.602Z 14 280-299 94 94 94 178 178 178 F F F
103902024-02-06T22:20:08.602Z 15 300-319 94 94 94 178 178 178 F F F
103912024-02-06T22:20:08.602Z 16 320-339 95 95 95 179 179 179 F F F
103922024-02-06T22:20:08.602Z 17 340-359 94 94 94 177 177 177 F F F
103932024-02-06T22:20:08.602Z 18 360-379 93 93 93 175 175 175 F F F
103942024-02-06T22:20:08.602Z 19 380-399 93 93 93 175 175 175 F F F
103952024-02-06T22:20:08.602Z 20 400-419 90 90 90 168 168 168 F F F
103962024-02-06T22:20:08.602Z 21 420-439 92 92 92 172 172 172 F F F
103972024-02-06T22:20:08.603Z 22 440-459 95 95 95 179 179 179 F F F
103982024-02-06T22:20:08.603Z 23 460-479 88 88 88 166 166 166 F F F
103992024-02-06T22:20:08.603Z 24 480-499 95 95 95 179 179 179 F F F
104002024-02-06T22:20:08.603Z 25 500-519 86 86 86 161 161 161 F F F
104012024-02-06T22:20:08.603Z 26 520-539 92 92 92 173 173 173 F F F
104022024-02-06T22:20:08.603Z 27 540-559 90 90 90 170 170 170 F F F
104032024-02-06T22:20:08.603Z 28 560-579 93 93 93 175 175 175 F F F
104042024-02-06T22:20:08.604Z 29 580-599 90 90 90 170 170 170 F F F
104052024-02-06T22:20:08.604ZMax gen: 95, Max flush: 179
104062024-02-06T22:20:08.604Z
104072024-02-06T22:20:08.604ZBegin loop 48
104082024-02-06T22:20:08.604ZDownstairs to restart: 0
104092024-02-06T22:20:08.604Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.020974516Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522}
104102024-02-06T22:20:08.605Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.02124513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522}
104112024-02-06T22:20:08.605ZCrucible runtime is spawned
104122024-02-06T22:20:08.605Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021289944Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522}
104132024-02-06T22:20:08.605Z{"msg":"Crucible 834a2eec-7877-4392-bdc3-b0086254ff87 has session id: d9e9bab6-2aea-4a3c-8f1f-bf57e637f925","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021324129Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104142024-02-06T22:20:08.605Z{"msg":"Upstairs opts: Upstairs UUID: 834a2eec-7877-4392-bdc3-b0086254ff87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021372483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104152024-02-06T22:20:08.606Z{"msg":"Crucible stats registered with UUID: 834a2eec-7877-4392-bdc3-b0086254ff87","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021399909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104162024-02-06T22:20:08.606Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021429685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522}
104172024-02-06T22:20:08.606Z{"msg":"834a2eec-7877-4392-bdc3-b0086254ff87 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021450932Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104182024-02-06T22:20:08.606Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021490947Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104192024-02-06T22:20:08.606Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021519163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104202024-02-06T22:20:08.606Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021555848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104212024-02-06T22:20:08.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021586894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104222024-02-06T22:20:08.607Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021663483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104232024-02-06T22:20:08.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.02169067Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104242024-02-06T22:20:08.607Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021717286Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104252024-02-06T22:20:08.607Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021747692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104262024-02-06T22:20:08.607Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021786337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104272024-02-06T22:20:08.607Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021812503Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104282024-02-06T22:20:08.607Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.02183652Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104292024-02-06T22:20:08.607Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.021894942Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104302024-02-06T22:20:08.607Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.022527587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104312024-02-06T22:20:08.608Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.022568881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104322024-02-06T22:20:08.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.022604926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104332024-02-06T22:20:08.608Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.022643351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104342024-02-06T22:20:08.608Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023248589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104352024-02-06T22:20:08.608Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023282825Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104362024-02-06T22:20:08.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023818802Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104372024-02-06T22:20:08.608Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023875325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104382024-02-06T22:20:08.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023901571Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104392024-02-06T22:20:08.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023930547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104402024-02-06T22:20:08.608Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023948725Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104412024-02-06T22:20:08.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023970262Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104422024-02-06T22:20:08.609Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.02398335Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104432024-02-06T22:20:08.609Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.023997448Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104442024-02-06T22:20:08.609Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.024486162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104452024-02-06T22:20:08.609Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.024932882Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104462024-02-06T22:20:08.609Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.024977806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104472024-02-06T22:20:08.609Z{"msg":"[0]R flush_numbers[0..12]: [179, 174, 168, 168, 173, 172, 172, 164, 179, 179, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025010511Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104482024-02-06T22:20:08.609Z{"msg":"[0]R generation[0..12]: [95, 92, 90, 90, 92, 92, 92, 88, 95, 95, 95, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025025889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104492024-02-06T22:20:08.609Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025049396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104502024-02-06T22:20:08.609Z{"msg":"[1]R flush_numbers[0..12]: [179, 174, 168, 168, 173, 172, 172, 164, 179, 179, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025064744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104512024-02-06T22:20:08.609Z{"msg":"[1]R generation[0..12]: [95, 92, 90, 90, 92, 92, 92, 88, 95, 95, 95, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025085781Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104522024-02-06T22:20:08.609Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025100429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104532024-02-06T22:20:08.609Z{"msg":"[2]R flush_numbers[0..12]: [179, 174, 168, 168, 173, 172, 172, 164, 179, 179, 179, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025121186Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104542024-02-06T22:20:08.610Z{"msg":"[2]R generation[0..12]: [95, 92, 90, 90, 92, 92, 92, 88, 95, 95, 95, 94]","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO
104552024-02-06T22:20:08.610Z:"2024-02-06T22:20:04.025135234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104562024-02-06T22:20:08.610Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025153322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104572024-02-06T22:20:08.610Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025174829Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104582024-02-06T22:20:08.610Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025189437Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104592024-02-06T22:20:08.610Z{"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025202955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104602024-02-06T22:20:08.610Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025216323Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104612024-02-06T22:20:08.610Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025236481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104622024-02-06T22:20:08.610Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025249069Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104632024-02-06T22:20:08.610Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025269476Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104642024-02-06T22:20:08.610Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025283105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104652024-02-06T22:20:08.610Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025296613Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104662024-02-06T22:20:08.611Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025309251Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104672024-02-06T22:20:08.611Z{"msg":"834a2eec-7877-4392-bdc3-b0086254ff87 is now active with session: d9e9bab6-2aea-4a3c-8f1f-bf57e637f925","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025329628Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104682024-02-06T22:20:08.611Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025341846Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522,"session_id":"d9e9bab6-2aea-4a3c-8f1f-bf57e637f925"}
104692024-02-06T22:20:08.611Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:04.025354395Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522}
104702024-02-06T22:20:08.611ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
104712024-02-06T22:20:08.611ZLoading write count information from file "/tmp/test_repair_verify.data"
104722024-02-06T22:20:08.611ZRun Repair workload
104732024-02-06T22:20:08.611Z01/30 Write block 245 len 28672 data: 6 6 11 11 12 10 7
104742024-02-06T22:20:08.656Z02/30 Flush
104752024-02-06T22:20:08.657Z03/30 Write block 86 len 16384 data: 5 5 5 6
104762024-02-06T22:20:08.657Z04/30 Write block 492 len 40960 data: 5 6 7 7 7 7 8 8 6 6
104772024-02-06T22:20:08.657Z05/30 Read block 160 len 8192
104782024-02-06T22:20:08.657Z06/30 Write block 85 len 32768 data: 6 6 6 6 7 6 7 4
104792024-02-06T22:20:08.657Z07/30 Read block 584 len 40960
104802024-02-06T22:20:08.658Z08/30 Write block 401 len 32768 data: 10 10 8 7 7 7 7 6
104812024-02-06T22:20:08.658Z09/30 Read block 316 len 36864
104822024-02-06T22:20:08.658Z10/30 Read block 200 len 24576
104832024-02-06T22:20:08.658Z11/30 Read block 127 len 32768
104842024-02-06T22:20:08.658Z12/30 Write block 452 len 36864 data: 3 3 4 5 4 5 4 4 5
104852024-02-06T22:20:08.658Z13/30 Flush
104862024-02-06T22:20:08.658Z14/30 Flush
104872024-02-06T22:20:08.658Z15/30 Write block 149 len 4096 data: 12
104882024-02-06T22:20:08.658Z16/30 Flush
104892024-02-06T22:20:08.659Z17/30 Write block 503 len 32768 data: 10 8 8 9 10 9 8 8
104902024-02-06T22:20:08.659Z18/30 Write block 590 len 20480 data: 13 12 11 12 11
104912024-02-06T22:20:08.659Z19/30 Write block 82 len 24576 data: 8 8 7 7 7 7
104922024-02-06T22:20:08.659Z20/30 Read block 123 len 40960
104932024-02-06T22:20:08.659Z21/30 Write block 475 len 24576 data: 9 9 10 10 8 5
104942024-02-06T22:20:08.659Z22/30 Flush
104952024-02-06T22:20:08.659Z23/30 Write block 204 len 8192 data: 10 9
104962024-02-06T22:20:08.659Z24/30 Write block 169 len 20480 data: 11 10 7 10 13
104972024-02-06T22:20:08.659Z25/30 Read block 442 len 40960
104982024-02-06T22:20:08.660Z26/30 Read block 47 len 24576
104992024-02-06T22:20:08.660Z27/30 Write block 95 len 12288 data: 6 6 6
105002024-02-06T22:20:08.660Z28/30 Write block 255 len 36864 data: 7 4 6 5 6 8 11 11 10
105012024-02-06T22:20:08.660Z29/30 Read block 38 len 16384
105022024-02-06T22:20:08.660Z30/30 Read block 454 len 28672
105032024-02-06T22:20:08.660Z----------------------------------------------------------------
105042024-02-06T22:20:08.660Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:28
105052024-02-06T22:20:08.661ZStates: Active Active Active
105062024-02-06T22:20:08.661ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
105072024-02-06T22:20:08.661Z 3 Acked 1002 Write 4 Done Done Done false
105082024-02-06T22:20:08.661Z 4 Acked 1003 Write 10 Sent Done Done false
105092024-02-06T22:20:08.661Z 5 Acked 1004 Read 2 Sent Done Done false
105102024-02-06T22:20:08.661Z 6 Acked 1005 Write 8 Sent Done Done false
105112024-02-06T22:20:08.661Z 7 Acked 1006 Read 10 Sent Done Done false
105122024-02-06T22:20:08.661Z 8 Acked 1007 Write 8 Sent Done Done false
105132024-02-06T22:20:08.661Z 9 Acked 1008 Read 9 Sent Done Done false
105142024-02-06T22:20:08.662Z 10 Acked 1009 Read 6 Sent Done Done false
105152024-02-06T22:20:08.662Z 11 Acked 1010 Read 8 Sent Done Done false
105162024-02-06T22:20:08.662Z 12 Acked 1011 Write 9 Sent Done Done false
105172024-02-06T22:20:08.662Z 13 Acked 1012 Flush 0 Sent Done Done false
105182024-02-06T22:20:08.662Z 14 Acked 1013 Flush 0 Sent Done Done false
105192024-02-06T22:20:08.662Z 15 Acked 1014 Write 1 Sent Done Done false
105202024-02-06T22:20:08.662Z 16 Acked 1015 Flush 0 Sent Done Done false
105212024-02-06T22:20:08.662Z 17 Acked 1016 Write 8 Sent Done Done false
105222024-02-06T22:20:08.663Z 18 Acked 1017 Write 5 Sent Done Done false
105232024-02-06T22:20:08.663Z 19 Acked 1018 Write 6 Sent Done Done false
105242024-02-06T22:20:08.663Z 20 Acked 1019 Read 10 Sent Done Done false
105252024-02-06T22:20:08.663Z 21 Acked 1020 Write 6 Sent Done Done false
105262024-02-06T22:20:08.663Z 22 Acked 1021 Flush 0 Sent Done Done false
105272024-02-06T22:20:08.663Z 23 Acked 1022 Write 2 Sent Done Done false
105282024-02-06T22:20:08.663Z 24 Acked 1023 Write 5 Sent Done Done false
105292024-02-06T22:20:08.663Z 25 Acked 1024 Read 10 Sent Done Done false
105302024-02-06T22:20:08.664Z 26 Acked 1025 Read 6 Sent Done Done false
105312024-02-06T22:20:08.664Z 27 Acked 1026 Write 3 Sent Done Done false
105322024-02-06T22:20:08.664Z 28 Acked 1027 Write 9 Sent Done Done false
105332024-02-06T22:20:08.664Z 29 Acked 1028 Read 4 Sent Done Done false
105342024-02-06T22:20:08.664Z 30 Acked 1029 Read 7 Sent Sent Done false
105352024-02-06T22:20:08.664Z STATES DS:0 DS:1 DS:2 TOTAL
105362024-02-06T22:20:08.664Z New 0 0 0 0
105372024-02-06T22:20:08.664Z Sent 27 1 0 28
105382024-02-06T22:20:08.664Z Done 1 27 28 56
105392024-02-06T22:20:08.664Z Skipped 0 0 0 0
105402024-02-06T22:20:08.664Z Error 0 0 0 0
105412024-02-06T22:20:08.664ZLast Flush: 1001 1021 1021
105422024-02-06T22:20:08.664ZDownstairs last five completed: 1001 1000
105432024-02-06T22:20:08.664ZUpstairs last five completed: 30 29 28 27 26
105442024-02-06T22:20:08.664Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:04.061121132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1522}
105452024-02-06T22:20:08.665ZWrote out file "/tmp/test_repair_verify.data"
105462024-02-06T22:20:08.665Z
105472024-02-06T22:20:08.665ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
105482024-02-06T22:20:08.665ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105492024-02-06T22:20:08.665Z 0 000-019 95 95 95 179 179 179 F F F
105502024-02-06T22:20:08.665Z 1 020-039 92 92 92 174 174 174 F F F
105512024-02-06T22:20:08.665Z 2 040-059 90 90 90 168 168 168 F F F
105522024-02-06T22:20:08.665Z 3 060-079 90 90 90 168 168 168 F F F
105532024-02-06T22:20:08.665Z 4 080-099 92 96 96 173 184 184 T T T <---
105542024-02-06T22:20:08.665Z 5 100-119 92 92 92 172 172 172 F F F
105552024-02-06T22:20:08.665Z 6 120-139 92 92 92 172 172 172 F F F
105562024-02-06T22:20:08.666Z 7 140-159 88 96 96 164 183 183 F F F <---
105572024-02-06T22:20:08.666Z 8 160-179 95 95 95 179 179 179 F T T <---
105582024-02-06T22:20:08.666Z 9 180-199 95 95 95 179 179 179 F F F
105592024-02-06T22:20:08.666Z 10 200-219 95 95 95 179 179 179 F T T <---
105602024-02-06T22:20:08.666Z 11 220-239 94 94 94 177 177 177 F F F
105612024-02-06T22:20:08.666Z 12 240-259 96 96 96 180 180 180 F T T <---
105622024-02-06T22:20:08.666Z 13 260-279 94 94 94 176 176 176 F T T <---
105632024-02-06T22:20:08.667Z 14 280-299 94 94 94 178 178 178 F F F
105642024-02-06T22:20:08.667Z 15 300-319 94 94 94 178 178 178 F F F
105652024-02-06T22:20:08.667Z 16 320-339 95 95 95 179 179 179 F F F
105662024-02-06T22:20:08.667Z 17 340-359 94 94 94 177 177 177 F F F
105672024-02-06T22:20:08.667Z 18 360-379 93 93 93 175 175 175 F F F
105682024-02-06T22:20:08.667Z 19 380-399 93 93 93 175 175 175 F F F
105692024-02-06T22:20:08.667Z 20 400-419 90 96 96 168 181 181 F F F <---
105702024-02-06T22:20:08.667Z 21 420-439 92 92 92 172 172 172 F F F
105712024-02-06T22:20:08.667Z 22 440-459 95 96 96 179 181 181 F F F <---
105722024-02-06T22:20:08.668Z 23 460-479 88 96 96 166 184 184 F F F <---
105732024-02-06T22:20:08.668Z 24 480-499 95 96 96 179 184 184 F F F <---
105742024-02-06T22:20:08.698Z 25 500-519 86 96 96 161 184 184 F F F <---
105752024-02-06T22:20:08.698Z 26 520-539 92 92 92 173 173 173 F F F
105762024-02-06T22:20:08.698Z 27 540-559 90 90 90 170 170 170 F F F
105772024-02-06T22:20:08.698Z 28 560-579 93 93 93 175 175 175 F F F
105782024-02-06T22:20:08.698Z 29 580-599 90 96 96 170 184 184 F F F <---
105792024-02-06T22:20:08.699ZMax gen: 96, Max flush: 184
105802024-02-06T22:20:08.699ZError: Difference in extent metadata found!
105812024-02-06T22:20:08.699ZA Difference in extent metadata is expected here
105822024-02-06T22:20:08.699ZOn loop 48
105832024-02-06T22:20:08.699Z
105842024-02-06T22:20:08.699ZVerifying data now
105852024-02-06T22:20:09.505ZNo change after verify
105862024-02-06T22:20:09.505ZLoop: 48 Downstairs dump after verify (and repair):
105872024-02-06T22:20:09.519ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105882024-02-06T22:20:09.520Z 0 000-019 95 95 95 179 179 179 F F F
105892024-02-06T22:20:09.520Z 1 020-039 92 92 92 174 174 174 F F F
105902024-02-06T22:20:09.520Z 2 040-059 90 90 90 168 168 168 F F F
105912024-02-06T22:20:09.520Z 3 060-079 90 90 90 168 168 168 F F F
105922024-02-06T22:20:09.520Z 4 080-099 97 97 97 185 185 185 F F F
105932024-02-06T22:20:09.521Z 5 100-119 92 92 92 172 172 172 F F F
105942024-02-06T22:20:09.521Z 6 120-139 92 92 92 172 172 172 F F F
105952024-02-06T22:20:09.521Z 7 140-159 96 96 96 183 183 183 F F F
105962024-02-06T22:20:09.521Z 8 160-179 97 97 97 185 185 185 F F F
105972024-02-06T22:20:09.535Z 9 180-199 95 95 95 179 179 179 F F F
105982024-02-06T22:20:09.535Z 10 200-219 97 97 97 185 185 185 F F F
105992024-02-06T22:20:09.535Z 11 220-239 94 94 94 177 177 177 F F F
106002024-02-06T22:20:09.535Z 12 240-259 97 97 97 185 185 185 F F F
106012024-02-06T22:20:09.535Z 13 260-279 97 97 97 185 185 185 F F F
106022024-02-06T22:20:09.535Z 14 280-299 94 94 94 178 178 178 F F F
106032024-02-06T22:20:09.536Z 15 300-319 94 94 94 178 178 178 F F F
106042024-02-06T22:20:09.536Z 16 320-339 95 95 95 179 179 179 F F F
106052024-02-06T22:20:09.536Z 17 340-359 94 94 94 177 177 177 F F F
106062024-02-06T22:20:09.536Z 18 360-379 93 93 93 175 175 175 F F F
106072024-02-06T22:20:09.536Z 19 380-399 93 93 93 175 175 175 F F F
106082024-02-06T22:20:09.536Z 20 400-419 96 96 96 181 181 181 F F F
106092024-02-06T22:20:09.537Z 21 420-439 92 92 92 172 172 172 F F F
106102024-02-06T22:20:09.537Z 22 440-459 96 96 96 181 181 181 F F F
106112024-02-06T22:20:09.537Z 23 460-479 96 96 96 184 184 184 F F F
106122024-02-06T22:20:09.537Z 24 480-499 96 96 96 184 184 184 F F F
106132024-02-06T22:20:09.537Z 25 500-519 96 96 96 184 184 184 F F F
106142024-02-06T22:20:09.537Z 26 520-539 92 92 92 173 173 173 F F F
106152024-02-06T22:20:09.537Z 27 540-559 90 90 90 170 170 170 F F F
106162024-02-06T22:20:09.538Z 28 560-579 93 93 93 175 175 175 F F F
106172024-02-06T22:20:09.538Z 29 580-599 96 96 96 184 184 184 F F F
106182024-02-06T22:20:09.538ZMax gen: 97, Max flush: 185
106192024-02-06T22:20:09.538Z
106202024-02-06T22:20:09.538ZBegin loop 49
106212024-02-06T22:20:09.538ZDownstairs to restart: 2
106222024-02-06T22:20:09.566Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.747600212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530}
106232024-02-06T22:20:09.566Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.74790853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530}
106242024-02-06T22:20:09.567Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.747943816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530}
106252024-02-06T22:20:09.567Z{"msg":"Crucible 5862e48d-838f-4475-b402-b7d40ecf9f82 has session id: 0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.747962133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106262024-02-06T22:20:09.567Z{"msg":"Upstairs opts: Upstairs UUID: 5862e48d-838f-4475-b402-b7d40ecf9f82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748014246Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106272024-02-06T22:20:09.567Z{"msg":"Crucible stats registered with UUID: 5862e48d-838f-4475-b402-b7d40ecf9f82","v":0,"Crucible runtime is spawned
106282024-02-06T22:20:09.567Zname":"crucible","level":30,"time":"2024-02-06T22:20:08.748072158Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106292024-02-06T22:20:09.567Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.74813117Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530}
106302024-02-06T22:20:09.567Z{"msg":"5862e48d-838f-4475-b402-b7d40ecf9f82 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748266232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106312024-02-06T22:20:09.567Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748314585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106322024-02-06T22:20:09.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748366328Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106332024-02-06T22:20:09.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748401914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106342024-02-06T22:20:09.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748436399Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106352024-02-06T22:20:09.568Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748481313Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106362024-02-06T22:20:09.568Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748517798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106372024-02-06T22:20:09.568Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748545344Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106382024-02-06T22:20:09.568Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.74857719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106392024-02-06T22:20:09.568Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748622864Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106402024-02-06T22:20:09.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748655689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106412024-02-06T22:20:09.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748692094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106422024-02-06T22:20:09.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.748729049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106432024-02-06T22:20:09.568Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.749496086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106442024-02-06T22:20:09.568Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.74953843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106452024-02-06T22:20:09.568Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.749564746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106462024-02-06T22:20:09.569Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.749603421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106472024-02-06T22:20:09.569Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750079927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106482024-02-06T22:20:09.569Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750219608Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106492024-02-06T22:20:09.569Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750268401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106502024-02-06T22:20:09.569Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750300007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106512024-02-06T22:20:09.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750560712Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106522024-02-06T22:20:09.569Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750600837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106532024-02-06T22:20:09.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750627743Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106542024-02-06T22:20:09.569Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750823886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106552024-02-06T22:20:09.569Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750891287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106562024-02-06T22:20:09.569Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750919443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106572024-02-06T22:20:09.569Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.750934221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106582024-02-06T22:20:09.570Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751323299Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106592024-02-06T22:20:09.570Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751370742Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106602024-02-06T22:20:09.570Z{"msg":"[0]R flush_numbers[0..12]: [179, 174, 168, 168, 185, 172, 172, 183, 185, 179, 185, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751400169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106612024-02-06T22:20:09.570Z{"msg":"[0]R generation[0..12]: [95, 92, 90, 90, 97, 92, 92, 96, 97, 95, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751432894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106622024-02-06T22:20:09.570Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.75146079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106632024-02-06T22:20:09.570Z{"msg":"[1]R flush_numbers[0..12]: [179, 174, 168, 168, 185, 172, 172, 183, 185, 179, 185, 177]","v":Wait for a query_work_queue command to finish before sending IO
106642024-02-06T22:20:09.570Z0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751495565Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106652024-02-06T22:20:09.570Z{"msg":"[1]R generation[0..12]: [95, 92, 90, 90, 97, 92, 92, 96, 97, 95, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751521862Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106662024-02-06T22:20:09.570Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751552938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106672024-02-06T22:20:09.570Z{"msg":"[2]R flush_numbers[0..12]: [179, 174, 168, 168, 185, 172, 172, 183, 185, 179, 185, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751578924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106682024-02-06T22:20:09.570Z{"msg":"[2]R generation[0..12]: [95, 92, 90, 90, 97, 92, 92, 96, 97, 95, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751604691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106692024-02-06T22:20:09.571Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751634757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106702024-02-06T22:20:09.571Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751659394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106712024-02-06T22:20:09.571Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.75168716Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106722024-02-06T22:20:09.571Z{"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751710956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106732024-02-06T22:20:09.571Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751737893Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106742024-02-06T22:20:09.571Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.7517617Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106752024-02-06T22:20:09.571Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751787716Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106762024-02-06T22:20:09.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751811673Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106772024-02-06T22:20:09.572ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106782024-02-06T22:20:09.572Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751841689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106792024-02-06T22:20:09.572Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751873365Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106802024-02-06T22:20:09.572Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751893552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106812024-02-06T22:20:09.572Z{"msg":"5862e48d-838f-4475-b402-b7d40ecf9f82 is now active with session: 0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751915989Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106822024-02-06T22:20:09.572Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751941135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530,"session_id":"0f195f2a-acb5-4c84-af5b-a7f5b9a5f1c9"}
106832024-02-06T22:20:09.572Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:08.751963232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530}
106842024-02-06T22:20:09.572ZLoading write count information from file "/tmp/test_repair_verify.data"
106852024-02-06T22:20:09.572ZRun Repair workload
106862024-02-06T22:20:09.572Z01/30 Write block 352 len 40960 data: 15 13 12 15 13 15 13 13 12 10
106872024-02-06T22:20:09.572Z02/30 Write block 571 len 16384 data: 12 13 12 13
106882024-02-06T22:20:09.572Z03/30 Write block 505 len 20480 data: 9 10 11 10 9
106892024-02-06T22:20:09.572Z04/30 Flush
106902024-02-06T22:20:10.004Z05/30 Write block 307 len 8192 data: 6 7
106912024-02-06T22:20:10.004Z06/30 Read block 408 len 16384
106922024-02-06T22:20:10.004Z07/30 Write block 303 len 12288 data: 10 8 10
106932024-02-06T22:20:10.004Z08/30 Write block 108 len 4096 data: 10
106942024-02-06T22:20:10.004Z09/30 Read block 367 len 12288
106952024-02-06T22:20:10.004Z10/30 Read block 206 len 20480
106962024-02-06T22:20:10.004Z11/30 Read block 472 len 36864
106972024-02-06T22:20:10.004Z12/30 Read block 550 len 32768
106982024-02-06T22:20:10.004Z13/30 Write block 71 len 24576 data: 5 6 7 9 8 5
106992024-02-06T22:20:10.004Z14/30 Write block 262 len 32768 data: 12 11 9 9 10 10 10 9
107002024-02-06T22:20:10.005Z15/30 Read block 394 len 40960
107012024-02-06T22:20:10.005Z16/30 Read block 1 len 32768
107022024-02-06T22:20:10.005Z17/30 Write block 555 len 40960 data: 5 7 7 8 5 5 5 6 5 8
107032024-02-06T22:20:10.005Z18/30 Flush
107042024-02-06T22:20:10.005Z19/30 Write block 517 len 4096 data: 10
107052024-02-06T22:20:10.005Z20/30 Write block 129 len 12288 data: 12 12 11
107062024-02-06T22:20:10.005Z21/30 Write block 566 len 24576 data: 9 11 11 9 9 13
107072024-02-06T22:20:10.005Z22/30 Write block 454 len 28672 data: 5 6 5 6 5 5 6
107082024-02-06T22:20:10.005Z23/30 Write block 361 len 40960 data: 11 14 13 11 10 8 10 11 10 8
107092024-02-06T22:20:10.005Z24/30 Write block 351 len 24576 data: 16 16 14 13 16 14
107102024-02-06T22:20:10.005Z25/30 Read block 149 len 28672
107112024-02-06T22:20:10.005Z26/30 Read block 264 len 16384
107122024-02-06T22:20:10.005Z27/30 Write block 552 len 32768 data: 7 7 6 6 8 8 9 6
107132024-02-06T22:20:10.005Z28/30 Write block 74 len 24576 data: 10 9 6 6 5 4
107142024-02-06T22:20:10.005Z29/30 Write block 405 len 8192 data: 8 8
107152024-02-06T22:20:10.005Z30/30 Read block 183 len 40960
107162024-02-06T22:20:10.005Z----------------------------------------------------------------
107172024-02-06T22:20:10.005Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
107182024-02-06T22:20:10.005ZStates: Active Active Active
107192024-02-06T22:20:10.005ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
107202024-02-06T22:20:10.006Z 1 Acked 1000 Write 10 Done Done Sent false
107212024-02-06T22:20:10.006Z 2 Acked 1001 Write 4 Done Done Sent false
107222024-02-06T22:20:10.006Z 3 Acked 1002 Write 5 Done Done Sent false
107232024-02-06T22:20:10.006Z 4 Acked 1003 Flush 0 Done Done Sent false
107242024-02-06T22:20:10.006Z 5 Acked 1004 Write 2 Done Done Sent false
107252024-02-06T22:20:10.006Z 6 Acked 1005 Read 4 Done Done Sent false
107262024-02-06T22:20:10.006Z 7 Acked 1006 Write 3 Done Done Sent false
107272024-02-06T22:20:10.006Z 8 Acked 1007 Write 1 Done Done Sent false
107282024-02-06T22:20:10.006Z 9 Acked 1008 Read 3 Done Done Sent false
107292024-02-06T22:20:10.006Z 10 Acked 1009 Read 5 Done Done Sent false
107302024-02-06T22:20:10.006Z 11 Acked 1010 Read 9 Done Done Sent false
107312024-02-06T22:20:10.006Z 12 Acked 1011 Read 8 Done Done Sent false
107322024-02-06T22:20:10.006Z 13 Acked 1012 Write 6 Done Done Sent false
107332024-02-06T22:20:10.006Z 14 Acked 1013 Write 8 Done Done Sent false
107342024-02-06T22:20:10.006Z 15 Acked 1014 Read 10 Done Done Sent false
107352024-02-06T22:20:10.006Z 16 Acked 1015 Read 8 Done Done Sent false
107362024-02-06T22:20:10.006Z 17 Acked 1016 Write 10 Done Done Sent false
107372024-02-06T22:20:10.006Z 18 Acked 1017 Flush 0 Done Done Sent false
107382024-02-06T22:20:10.006Z 19 Acked 1018 Write 1 Done Done Sent false
107392024-02-06T22:20:10.006Z 20 Acked 1019 Write 3 Done Done Sent false
107402024-02-06T22:20:10.006Z 21 Acked 1020 Write 6 Done Done Sent false
107412024-02-06T22:20:10.006Z 22 Acked 1021 Write 7 Done Done Sent false
107422024-02-06T22:20:10.006Z 23 Acked 1022 Write 10 Done Done Sent false
107432024-02-06T22:20:10.006Z 24 Acked 1023 Write 6 Done Done Sent false
107442024-02-06T22:20:10.007Z 25 Acked 1024 Read 7 Done Done Sent false
107452024-02-06T22:20:10.007Z 26 Acked 1025 Read 4 Done Done Sent false
107462024-02-06T22:20:10.007Z 27 Acked 1026 Write 8 Done Done Sent false
107472024-02-06T22:20:10.007Z 28 Acked 1027 Write 6 Done Done Sent false
107482024-02-06T22:20:10.007Z 29 Acked 1028 Write 2 Done Done Sent false
107492024-02-06T22:20:10.007Z 30 Acked 1029 Read 10 Done Sent Sent false
107502024-02-06T22:20:10.007Z STATES DS:0 DS:1 DS:2 TOTAL
107512024-02-06T22:20:10.007Z New 0 0 0 0
107522024-02-06T22:20:10.007Z Sent 0 1 30 31
107532024-02-06T22:20:10.007Z Done 30 29 0 59
107542024-02-06T22:20:10.007Z Skipped 0 0 0 0
107552024-02-06T22:20:10.007Z Error 0 0 0 0
107562024-02-06T22:20:10.007ZLast Flush: 1017 1017 0
107572024-02-06T22:20:10.007ZDownstairs last five completed:
107582024-02-06T22:20:10.007ZUpstairs last five completed: 30 29 28 27 26
107592024-02-06T22:20:10.007Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:08.782722386Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1530}
107602024-02-06T22:20:10.007ZWrote out file "/tmp/test_repair_verify.data"
107612024-02-06T22:20:10.007Z
107622024-02-06T22:20:10.033ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
107632024-02-06T22:20:10.034ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107642024-02-06T22:20:10.034Z 0 000-019 95 95 95 179 179 179 F F F
107652024-02-06T22:20:10.034Z 1 020-039 92 92 92 174 174 174 F F F
107662024-02-06T22:20:10.034Z 2 040-059 90 90 90 168 168 168 F F F
107672024-02-06T22:20:10.034Z 3 060-079 98 98 90 187 187 168 T T F <---
107682024-02-06T22:20:10.034Z 4 080-099 97 97 97 185 185 185 F F F
107692024-02-06T22:20:10.035Z 5 100-119 98 98 92 187 187 172 F F F <---
107702024-02-06T22:20:10.035Z 6 120-139 92 92 92 172 172 172 T T F <---
107712024-02-06T22:20:10.035Z 7 140-159 96 96 96 183 183 183 F F F
107722024-02-06T22:20:10.035Z 8 160-179 97 97 97 185 185 185 F F F
107732024-02-06T22:20:10.035Z 9 180-199 95 95 95 179 179 179 F F F
107742024-02-06T22:20:10.035Z 10 200-219 97 97 97 185 185 185 F F F
107752024-02-06T22:20:10.036Z 11 220-239 94 94 94 177 177 177 F F F
107762024-02-06T22:20:10.036Z 12 240-259 97 97 97 185 185 185 F F F
107772024-02-06T22:20:10.036Z 13 260-279 98 98 97 187 187 185 F F F <---
107782024-02-06T22:20:10.036Z 14 280-299 94 94 94 178 178 178 F F F
107792024-02-06T22:20:10.036Z 15 300-319 98 98 94 187 187 178 F F F <---
107802024-02-06T22:20:10.036Z 16 320-339 95 95 95 179 179 179 F F F
107812024-02-06T22:20:10.036Z 17 340-359 98 98 94 186 186 177 T T F <---
107822024-02-06T22:20:10.037Z 18 360-379 98 98 93 186 186 175 T T F <---
107832024-02-06T22:20:10.037Z 19 380-399 93 93 93 175 175 175 F F F
107842024-02-06T22:20:10.037Z 20 400-419 96 96 96 181 181 181 T T F <---
107852024-02-06T22:20:10.037Z 21 420-439 92 92 92 172 172 172 F F F
107862024-02-06T22:20:10.037Z 22 440-459 96 96 96 181 181 181 T T F <---
107872024-02-06T22:20:10.037Z 23 460-479 96 96 96 184 184 184 T T F <---
107882024-02-06T22:20:10.037Z 24 480-499 96 96 96 184 184 184 F F F
107892024-02-06T22:20:10.038Z 25 500-519 98 98 96 186 186 184 T T F <---
107902024-02-06T22:20:10.038Z 26 520-539 92 92 92 173 173 173 F F F
107912024-02-06T22:20:10.038Z 27 540-559 98 98 90 187 187 170 T T F <---
107922024-02-06T22:20:10.038Z 28 560-579 98 98 93 187 187 175 T T F <---
107932024-02-06T22:20:10.038Z 29 580-599 96 96 96 184 184 184 F F F
107942024-02-06T22:20:10.038ZMax gen: 98, Max flush: 187
107952024-02-06T22:20:10.038ZError: Difference in extent metadata found!
107962024-02-06T22:20:10.038ZA Difference in extent metadata is expected here
107972024-02-06T22:20:10.039ZOn loop 49
107982024-02-06T22:20:10.039Z
107992024-02-06T22:20:10.039ZVerifying data now
108002024-02-06T22:20:14.264ZNo change after verify
108012024-02-06T22:20:14.264ZLoop: 49 Downstairs dump after verify (and repair):
108022024-02-06T22:20:14.278ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108032024-02-06T22:20:14.278Z 0 000-019 95 95 95 179 179 179 F F F
108042024-02-06T22:20:14.278Z 1 020-039 92 92 92 174 174 174 F F F
108052024-02-06T22:20:14.278Z 2 040-059 90 90 90 168 168 168 F F F
108062024-02-06T22:20:14.282Z 3 060-079 99 99 99 188 188 188 F F F
108072024-02-06T22:20:14.282Z 4 080-099 97 97 97 185 185 185 F F F
108082024-02-06T22:20:14.282Z 5 100-119 98 98 98 187 187 187 F F F
108092024-02-06T22:20:14.282Z 6 120-139 99 99 99 188 188 188 F F F
108102024-02-06T22:20:14.282Z 7 140-159 96 96 96 183 183 183 F F F
108112024-02-06T22:20:14.282Z 8 160-179 97 97 97 185 185 185 F F F
108122024-02-06T22:20:14.282Z 9 180-199 95 95 95 179 179 179 F F F
108132024-02-06T22:20:14.282Z 10 200-219 97 97 97 185 185 185 F F F
108142024-02-06T22:20:14.282Z 11 220-239 94 94 94 177 177 177 F F F
108152024-02-06T22:20:14.282Z 12 240-259 97 97 97 185 185 185 F F F
108162024-02-06T22:20:14.282Z 13 260-279 98 98 98 187 187 187 F F F
108172024-02-06T22:20:14.283Z 14 280-299 94 94 94 178 178 178 F F F
108182024-02-06T22:20:14.283Z 15 300-319 98 98 98 187 187 187 F F F
108192024-02-06T22:20:14.283Z 16 320-339 95 95 95 179 179 179 F F F
108202024-02-06T22:20:14.283Z 17 340-359 99 99 99 188 188 188 F F F
108212024-02-06T22:20:14.283Z 18 360-379 99 99 99 188 188 188 F F F
108222024-02-06T22:20:14.283Z 19 380-399 93 93 93 175 175 175 F F F
108232024-02-06T22:20:14.283Z 20 400-419 99 99 99 188 188 188 F F F
108242024-02-06T22:20:14.283Z 21 420-439 92 92 92 172 172 172 F F F
108252024-02-06T22:20:14.283Z 22 440-459 99 99 99 188 188 188 F F F
108262024-02-06T22:20:14.283Z 23 460-479 99 99 99 188 188 188 F F F
108272024-02-06T22:20:14.283Z 24 480-499 96 96 96 184 184 184 F F F
108282024-02-06T22:20:14.283Z 25 500-519 99 99 99 188 188 188 F F F
108292024-02-06T22:20:14.283Z 26 520-539 92 92 92 173 173 173 F F F
108302024-02-06T22:20:14.283Z 27 540-559 99 99 99 188 188 188 F F F
108312024-02-06T22:20:14.283Z 28 560-579 99 99 99 188 188 188 F F F
108322024-02-06T22:20:14.283Z 29 580-599 96 96 96 184 184 184 F F F
108332024-02-06T22:20:14.283ZMax gen: 99, Max flush: 188
108342024-02-06T22:20:14.287Z
108352024-02-06T22:20:14.287ZBegin loop 50
108362024-02-06T22:20:14.287ZDownstairs to restart: 0
108372024-02-06T22:20:14.302Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505424173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538}
108382024-02-06T22:20:14.302Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505690067Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538}
108392024-02-06T22:20:14.309Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505725982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538}
108402024-02-06T22:20:14.309Z{"msg":"Crucible b6181caf-b8c0-4e23-b52c-13b23214ebe3 has session id: 3441a3de-c7f5-4acd-a942-2576f92fea32","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505751189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108412024-02-06T22:20:14.309Z{"msg":"Upstairs opts: Upstairs UUID: b6181caf-b8c0-4e23-b52c-13b23214ebe3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505799792Z","hostname":"Crucible runtime is spawned
108422024-02-06T22:20:14.309Zip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108432024-02-06T22:20:14.309Z{"msg":"Crucible stats registered with UUID: b6181caf-b8c0-4e23-b52c-13b23214ebe3","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505836397Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108442024-02-06T22:20:14.309Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505883031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538}
108452024-02-06T22:20:14.310Z{"msg":"b6181caf-b8c0-4e23-b52c-13b23214ebe3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.50596341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108462024-02-06T22:20:14.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.505989776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108472024-02-06T22:20:14.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506005544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108482024-02-06T22:20:14.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506019562Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108492024-02-06T22:20:14.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506052658Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108502024-02-06T22:20:14.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506078204Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108512024-02-06T22:20:14.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506094142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108522024-02-06T22:20:14.310Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506175071Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108532024-02-06T22:20:14.310Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506234263Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108542024-02-06T22:20:14.310Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506288926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108552024-02-06T22:20:14.310Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506323851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108562024-02-06T22:20:14.310Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506418338Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108572024-02-06T22:20:14.310Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.506472631Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108582024-02-06T22:20:14.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.507036055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108592024-02-06T22:20:14.310Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.507089568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108602024-02-06T22:20:14.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.507120603Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108612024-02-06T22:20:14.310Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.50714569Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108622024-02-06T22:20:14.310Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.507688837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108632024-02-06T22:20:14.310Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.507847096Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108642024-02-06T22:20:14.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508193869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108652024-02-06T22:20:14.310Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508231164Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108662024-02-06T22:20:14.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508248861Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108672024-02-06T22:20:14.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508365895Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108682024-02-06T22:20:14.310Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508414479Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108692024-02-06T22:20:14.311Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508501697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108702024-02-06T22:20:14.311Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508538912Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108712024-02-06T22:20:14.311Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.5085557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108722024-02-06T22:20:14.311Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.508886225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108732024-02-06T22:20:14.311Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509310338Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108742024-02-06T22:20:14.311Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509386068Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108752024-02-06T22:20:14.311Z{"msg":"[0]R flush_numbers[0..12]: [179, 174, 168, 188, 185, 187, 188, 183, 185, 179, 185, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509425342Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108762024-02-06T22:20:14.311Z{"msg":"[0]R generation[0..12]: [95, 92, 90, 99, 97, 98, 99, 96, 97, 95, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509460837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108772024-02-06T22:20:14.311Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509498282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108782024-02-06T22:20:14.311Z{"msg":"[1]R flush_numbers[0..12]: [179, 174, 168, 188, 185, 187, 188, 183, 185, 179, 185, 177]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
108792024-02-06T22:20:14.311Z,"time":"2024-02-06T22:20:13.509536737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108802024-02-06T22:20:14.311Z{"msg":"[1]R generation[0..12]: [95, 92, 90, 99, 97, 98, 99, 96, 97, 95, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509578491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108812024-02-06T22:20:14.311Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509619086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108822024-02-06T22:20:14.311Z{"msg":"[2]R flush_numbers[0..12]: [179, 174, 168, 188, 185, 187, 188, 183, 185, 179, 185, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509657811Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108832024-02-06T22:20:14.311Z{"msg":"[2]R generation[0..12]: [95, 92, 90, 99, 97, 98, 99, 96, 97, 95, 97, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509696366Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108842024-02-06T22:20:14.311Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.50973696Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108852024-02-06T22:20:14.311Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509774375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108862024-02-06T22:20:14.311Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.50981204Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108872024-02-06T22:20:14.311Z{"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509847745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108882024-02-06T22:20:14.311Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108892024-02-06T22:20:14.311Z,"time":"2024-02-06T22:20:13.50988394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108902024-02-06T22:20:14.311Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509920975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108912024-02-06T22:20:14.311Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509948232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108922024-02-06T22:20:14.311Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509964279Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108932024-02-06T22:20:14.312Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.509986266Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108942024-02-06T22:20:14.312Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.510022371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108952024-02-06T22:20:14.312Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.51003728Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108962024-02-06T22:20:14.312Z{"msg":"b6181caf-b8c0-4e23-b52c-13b23214ebe3 is now active with session: 3441a3de-c7f5-4acd-a942-2576f92fea32","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.510057907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108972024-02-06T22:20:14.312Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.510070175Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538,"session_id":"3441a3de-c7f5-4acd-a942-2576f92fea32"}
108982024-02-06T22:20:14.312Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:13.510083663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538}
108992024-02-06T22:20:14.312ZLoading write count information from file "/tmp/test_repair_verify.data"
109002024-02-06T22:20:14.312ZRun Repair workload
109012024-02-06T22:20:14.312Z01/30 Write block 341 len 36864 data: 6 8 6 7 8 10 11 13 12
109022024-02-06T22:20:14.315Z02/30 Write block 406 len 16384 data: 9 8 7 8
109032024-02-06T22:20:14.315Z03/30 Read block 422 len 28672
109042024-02-06T22:20:14.315Z04/30 Write block 123 len 12288 data: 11 11 12
109052024-02-06T22:20:14.315Z05/30 Read block 137 len 32768
109062024-02-06T22:20:14.316Z06/30 Write block 194 len 20480 data: 10 9 12 11 8
109072024-02-06T22:20:14.316Z07/30 Read block 550 len 24576
109082024-02-06T22:20:14.316Z08/30 Write block 284 len 12288 data: 7 5 5
109092024-02-06T22:20:14.316Z09/30 Flush
109102024-02-06T22:20:14.316Z10/30 Write block 321 len 8192 data: 6 7
109112024-02-06T22:20:14.316Z11/30 Read block 160 len 24576
109122024-02-06T22:20:14.319Z12/30 Flush
109132024-02-06T22:20:14.323Z13/30 Write block 395 len 20480 data: 7 6 7 7 8
109142024-02-06T22:20:14.323Z14/30 Write block 315 len 12288 data: 12 11 9
109152024-02-06T22:20:14.323Z15/30 Write block 149 len 36864 data: 13 10 8 8 11 10 11 9 11
109162024-02-06T22:20:14.323Z16/30 Flush
109172024-02-06T22:20:14.326Z17/30 Write block 400 len 40960 data: 9 11 11 9 8 9 10 9 8 9
109182024-02-06T22:20:14.326Z18/30 Write block 556 len 32768 data: 9 9 10 7 6 6 7 6
109192024-02-06T22:20:14.326Z19/30 Write block 458 len 40960 data: 6 6 7 5 5 5 5 5 4 8
109202024-02-06T22:20:14.330Z20/30 Read block 580 len 12288
109212024-02-06T22:20:14.330Z21/30 Write block 258 len 40960 data: 6 7 9 12 13 12 10 10 11 11
109222024-02-06T22:20:14.330Z22/30 Read block 242 len 40960
109232024-02-06T22:20:14.335Z23/30 Write block 586 len 20480 data: 6 9 9 11 14
109242024-02-06T22:20:14.335Z24/30 Write block 420 len 36864 data: 11 9 7 8 9 8 9 7 8
109252024-02-06T22:20:14.335Z25/30 Write block 214 len 20480 data: 10 10 8 8 9
109262024-02-06T22:20:14.335Z26/30 Read block 129 len 8192
109272024-02-06T22:20:14.335Z27/30 Read block 557 len 28672
109282024-02-06T22:20:14.339Z28/30 Write block 164 len 4096 data: 6
109292024-02-06T22:20:14.340Z29/30 Read block 45 len 16384
109302024-02-06T22:20:14.340Z30/30 Write block 460 len 12288 data: 8 6 6
109312024-02-06T22:20:14.340Z----------------------------------------------------------------
109322024-02-06T22:20:14.340Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
109332024-02-06T22:20:14.340ZStates: Active Active Active
109342024-02-06T22:20:14.340ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109352024-02-06T22:20:14.340Z 1 Acked 1000 Write 9 Done Done Done false
109362024-02-06T22:20:14.340Z 2 Acked 1001 Write 4 Done Done Done false
109372024-02-06T22:20:14.340Z 3 Acked 1002 Read 7 Done Done Done false
109382024-02-06T22:20:14.340Z 4 Acked 1003 Write 3 Done Done Done false
109392024-02-06T22:20:14.340Z 5 Acked 1004 Read 8 Done Done Done false
109402024-02-06T22:20:14.340Z 6 Acked 1005 Write 5 Done Done Done false
109412024-02-06T22:20:14.340Z 7 Acked 1006 Read 6 Done Done Done false
109422024-02-06T22:20:14.340Z 8 Acked 1007 Write 3 Sent Done Done false
109432024-02-06T22:20:14.340Z 9 Acked 1008 Flush 0 Sent Done Done false
109442024-02-06T22:20:14.340Z 10 Acked 1009 Write 2 Sent Done Done false
109452024-02-06T22:20:14.340Z 11 Acked 1010 Read 6 Sent Done Done false
109462024-02-06T22:20:14.340Z 12 Acked 1011 Flush 0 Sent Done Done false
109472024-02-06T22:20:14.340Z 13 Acked 1012 Write 5 Sent Done Done false
109482024-02-06T22:20:14.340Z 14 Acked 1013 Write 3 Sent Done Done false
109492024-02-06T22:20:14.340Z 15 Acked 1014 Write 9 Sent Done Done false
109502024-02-06T22:20:14.340Z 16 Acked 1015 Flush 0 Sent Done Done false
109512024-02-06T22:20:14.340Z 17 Acked 1016 Write 10 Sent Done Done false
109522024-02-06T22:20:14.340Z 18 Acked 1017 Write 8 Sent Done Done false
109532024-02-06T22:20:14.341Z 19 Acked 1018 Write 10 Sent Done Done false
109542024-02-06T22:20:14.341Z 20 Acked 1019 Read 3 Sent Done Done false
109552024-02-06T22:20:14.341Z 21 Acked 1020 Write 10 Sent Done Done false
109562024-02-06T22:20:14.341Z 22 Acked 1021 Read 10 Sent Done Done false
109572024-02-06T22:20:14.341Z 23 Acked 1022 Write 5 Sent Done Done false
109582024-02-06T22:20:14.341Z 24 Acked 1023 Write 9 Sent Done Done false
109592024-02-06T22:20:14.341Z 25 Acked 1024 Write 5 Sent Done Done false
109602024-02-06T22:20:14.341Z 26 Acked 1025 Read 2 Sent Done Done false
109612024-02-06T22:20:14.341Z 27 Acked 1026 Read 7 Sent Done Done false
109622024-02-06T22:20:14.341Z 28 Acked 1027 Write 1 Sent Done Done false
109632024-02-06T22:20:14.341Z 29 Acked 1028 Read 4 Sent Done Done false
109642024-02-06T22:20:14.341Z 30 Acked 1029 Write 3 Sent Sent Sent false
109652024-02-06T22:20:14.341Z STATES DS:0 DS:1 DS:2 TOTAL
109662024-02-06T22:20:14.341Z New 0 0 0 0
109672024-02-06T22:20:14.341Z Sent 23 1 1 25
109682024-02-06T22:20:14.341Z Done 7 29 29 65
109692024-02-06T22:20:14.341Z Skipped 0 0 0 0
109702024-02-06T22:20:14.341Z Error 0 0 0 0
109712024-02-06T22:20:14.341ZLast Flush: 0 1015 1015
109722024-02-06T22:20:14.341ZDownstairs last five completed:
109732024-02-06T22:20:14.341ZUpstairs last five completed: 30 29 28 27 26
109742024-02-06T22:20:14.341Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:13.542326657Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1538}
109752024-02-06T22:20:14.341ZWrote out file "/tmp/test_repair_verify.data"
109762024-02-06T22:20:14.345Z
109772024-02-06T22:20:14.345ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109782024-02-06T22:20:14.364ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109792024-02-06T22:20:14.364Z 0 000-019 95 95 95 179 179 179 F F F
109802024-02-06T22:20:14.364Z 1 020-039 92 92 92 174 174 174 F F F
109812024-02-06T22:20:14.364Z 2 040-059 90 90 90 168 168 168 F F F
109822024-02-06T22:20:14.364Z 3 060-079 99 99 99 188 188 188 F F F
109832024-02-06T22:20:14.364Z 4 080-099 97 97 97 185 185 185 F F F
109842024-02-06T22:20:14.364Z 5 100-119 98 98 98 187 187 187 F F F
109852024-02-06T22:20:14.364Z 6 120-139 99 100 100 188 189 189 T F F <---
109862024-02-06T22:20:14.365Z 7 140-159 96 100 100 183 191 191 F F F <---
109872024-02-06T22:20:14.365Z 8 160-179 97 97 97 185 185 185 F T T <---
109882024-02-06T22:20:14.365Z 9 180-199 95 100 100 179 189 189 T F F <---
109892024-02-06T22:20:14.365Z 10 200-219 97 97 97 185 185 185 F T T <---
109902024-02-06T22:20:14.370Z 11 220-239 94 94 94 177 177 177 F F F
109912024-02-06T22:20:14.370Z 12 240-259 97 97 97 185 185 185 F T T <---
109922024-02-06T22:20:14.370Z 13 260-279 98 98 98 187 187 187 F T T <---
109932024-02-06T22:20:14.370Z 14 280-299 94 100 100 178 189 189 F F F <---
109942024-02-06T22:20:14.370Z 15 300-319 98 100 100 187 191 191 F F F <---
109952024-02-06T22:20:14.370Z 16 320-339 95 100 100 179 190 190 F F F <---
109962024-02-06T22:20:14.370Z 17 340-359 99 100 100 188 189 189 T F F <---
109972024-02-06T22:20:14.370Z 18 360-379 99 99 99 188 188 188 F F F
109982024-02-06T22:20:14.370Z 19 380-399 93 100 100 175 191 191 F F F <---
109992024-02-06T22:20:14.370Z 20 400-419 99 100 100 188 189 189 T T T <---
110002024-02-06T22:20:14.370Z 21 420-439 92 92 92 172 172 172 F T T <---
110012024-02-06T22:20:14.370Z 22 440-459 99 99 99 188 188 188 F T T <---
110022024-02-06T22:20:14.370Z 23 460-479 99 99 99 188 188 188 F T T <---
110032024-02-06T22:20:14.370Z 24 480-499 96 96 96 184 184 184 F F F
110042024-02-06T22:20:14.371Z 25 500-519 99 99 99 188 188 188 F F F
110052024-02-06T22:20:14.371Z 26 520-539 92 92 92 173 173 173 F F F
110062024-02-06T22:20:14.371Z 27 540-559 99 99 99 188 188 188 F T T <---
110072024-02-06T22:20:14.371Z 28 560-579 99 99 99 188 188 188 F T T <---
110082024-02-06T22:20:14.371Z 29 580-599 96 96 96 184 184 184 F T T <---
110092024-02-06T22:20:14.371ZMax gen: 100, Max flush: 191
110102024-02-06T22:20:14.371ZError: Difference in extent metadata found!
110112024-02-06T22:20:14.371ZA Difference in extent metadata is expected here
110122024-02-06T22:20:14.371ZOn loop 50
110132024-02-06T22:20:14.371Z
110142024-02-06T22:20:14.375ZVerifying data now
110152024-02-06T22:20:20.263ZNo change after verify
110162024-02-06T22:20:20.263ZLoop: 50 Downstairs dump after verify (and repair):
110172024-02-06T22:20:20.278ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110182024-02-06T22:20:20.278Z 0 000-019 95 95 95 179 179 179 F F F
110192024-02-06T22:20:20.278Z 1 020-039 92 92 92 174 174 174 F F F
110202024-02-06T22:20:20.279Z 2 040-059 90 90 90 168 168 168 F F F
110212024-02-06T22:20:20.279Z 3 060-079 99 99 99 188 188 188 F F F
110222024-02-06T22:20:20.279Z 4 080-099 97 97 97 185 185 185 F F F
110232024-02-06T22:20:20.284Z 5 100-119 98 98 98 187 187 187 F F F
110242024-02-06T22:20:20.284Z 6 120-139 100 100 100 189 189 189 F F F
110252024-02-06T22:20:20.284Z 7 140-159 100 100 100 191 191 191 F F F
110262024-02-06T22:20:20.284Z 8 160-179 101 101 101 192 192 192 F F F
110272024-02-06T22:20:20.284Z 9 180-199 100 100 100 189 189 189 F F F
110282024-02-06T22:20:20.284Z 10 200-219 101 101 101 192 192 192 F F F
110292024-02-06T22:20:20.284Z 11 220-239 94 94 94 177 177 177 F F F
110302024-02-06T22:20:20.284Z 12 240-259 101 101 101 192 192 192 F F F
110312024-02-06T22:20:20.284Z 13 260-279 101 101 101 192 192 192 F F F
110322024-02-06T22:20:20.284Z 14 280-299 100 100 100 189 189 189 F F F
110332024-02-06T22:20:20.284Z 15 300-319 100 100 100 191 191 191 F F F
110342024-02-06T22:20:20.284Z 16 320-339 100 100 100 190 190 190 F F F
110352024-02-06T22:20:20.284Z 17 340-359 100 100 100 189 189 189 F F F
110362024-02-06T22:20:20.284Z 18 360-379 99 99 99 188 188 188 F F F
110372024-02-06T22:20:20.284Z 19 380-399 100 100 100 191 191 191 F F F
110382024-02-06T22:20:20.284Z 20 400-419 101 101 101 192 192 192 F F F
110392024-02-06T22:20:20.284Z 21 420-439 101 101 101 192 192 192 F F F
110402024-02-06T22:20:20.284Z 22 440-459 101 101 101 192 192 192 F F F
110412024-02-06T22:20:20.284Z 23 460-479 101 101 101 192 192 192 F F F
110422024-02-06T22:20:20.284Z 24 480-499 96 96 96 184 184 184 F F F
110432024-02-06T22:20:20.284Z 25 500-519 99 99 99 188 188 188 F F F
110442024-02-06T22:20:20.285Z 26 520-539 92 92 92 173 173 173 F F F
110452024-02-06T22:20:20.285Z 27 540-559 101 101 101 192 192 192 F F F
110462024-02-06T22:20:20.285Z 28 560-579 101 101 101 192 192 192 F F F
110472024-02-06T22:20:20.285Z 29 580-599 101 101 101 192 192 192 F F F
110482024-02-06T22:20:20.285ZMax gen: 101, Max flush: 192
110492024-02-06T22:20:20.285Z
110502024-02-06T22:20:20.285ZBegin loop 51
110512024-02-06T22:20:20.285ZDownstairs to restart: 0
110522024-02-06T22:20:20.308Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.504805614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546}
110532024-02-06T22:20:20.308Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505103174Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":Crucible runtime is spawned
110542024-02-06T22:20:20.308Z1546}
110552024-02-06T22:20:20.341Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505144858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546}
110562024-02-06T22:20:20.341Z{"msg":"Crucible 2b222732-e238-423b-a4d3-ad22be87728d has session id: 7903e05d-4f87-4b48-83df-b3324c78742f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505175074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110572024-02-06T22:20:20.341Z{"msg":"Upstairs opts: Upstairs UUID: 2b222732-e238-423b-a4d3-ad22be87728d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505229527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110582024-02-06T22:20:20.341Z{"msg":"Crucible stats registered with UUID: 2b222732-e238-423b-a4d3-ad22be87728d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505274291Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110592024-02-06T22:20:20.341Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505315126Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546}
110602024-02-06T22:20:20.341Z{"msg":"2b222732-e238-423b-a4d3-ad22be87728d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505339462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110612024-02-06T22:20:20.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505380077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110622024-02-06T22:20:20.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505415642Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110632024-02-06T22:20:20.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505440278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110642024-02-06T22:20:20.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505465115Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110652024-02-06T22:20:20.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.50550432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110662024-02-06T22:20:20.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505543745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110672024-02-06T22:20:20.341Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505569261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110682024-02-06T22:20:20.341Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505599147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110692024-02-06T22:20:20.341Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505626424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110702024-02-06T22:20:20.341Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:19.505668528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110712024-02-06T22:20:20.342Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:19.505708702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110722024-02-06T22:20:20.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505738918Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110732024-02-06T22:20:20.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505772424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110742024-02-06T22:20:20.342Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:19.505862112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110752024-02-06T22:20:20.342Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505899407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110762024-02-06T22:20:20.342Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.505916824Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110772024-02-06T22:20:20.342Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.506077383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110782024-02-06T22:20:20.342Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.506512554Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110792024-02-06T22:20:20.342Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.506554798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110802024-02-06T22:20:20.342Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.506571086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110812024-02-06T22:20:20.342Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.506605161Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110822024-02-06T22:20:20.342Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507231976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110832024-02-06T22:20:20.342Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507275151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110842024-02-06T22:20:20.342Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507749777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110852024-02-06T22:20:20.342Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507803419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110862024-02-06T22:20:20.342Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507848873Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110872024-02-06T22:20:20.342Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507887608Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110882024-02-06T22:20:20.342Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507911005Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110892024-02-06T22:20:20.342Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:19.507940491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110902024-02-06T22:20:21.302Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.506383349Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110912024-02-06T22:20:21.307Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.506587302Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110922024-02-06T22:20:21.307Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.507545422Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110932024-02-06T22:20:21.307Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.507582047Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110942024-02-06T22:20:21.307Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508021938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110952024-02-06T22:20:21.307Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508510082Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110962024-02-06T22:20:21.307Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508554126Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110972024-02-06T22:20:21.307Z{"msg":"[0]R flush_numbers[0..12]: [179, 174, 168, 188, 185, 187, 189, 191, 192, 189, 192, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508575273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110982024-02-06T22:20:21.307Z{"msg":"[0]R generation[0..12]: [95, 92, 90, 99, 97, 98, 100, 100, 101, 100, 101, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508617257Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
110992024-02-06T22:20:21.307Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508646743Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111002024-02-06T22:20:21.307Z{"msg":"[1]R flush_numbers[0..12]: [179, 174, 168, 188, 185, 187, 189, 191, 192, 189, 192, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508683668Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111012024-02-06T22:20:21.307ZWait for a query_work_queue command to finish before sending IO
111022024-02-06T22:20:21.308Z{"msg":"[1]R generation[0..12]: [95, 92, 90, 99, 97, 98, 100, 100, 101, 100, 101, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508717784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111032024-02-06T22:20:21.308Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508753189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111042024-02-06T22:20:21.308Z{"msg":"[2]R flush_numbers[0..12]: [179, 174, 168, 188, 185, 187, 189, 191, 192, 189, 192, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508800843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111052024-02-06T22:20:21.308Z{"msg":"[2]R generation[0..12]: [95, 92, 90, 99, 97, 98, 100, 100, 101, 100, 101, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508834558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111062024-02-06T22:20:21.308Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508876512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111072024-02-06T22:20:21.308Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508907278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111082024-02-06T22:20:21.308Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.508952462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111092024-02-06T22:20:21.308Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509025392Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111102024-02-06T22:20:21.308Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509065477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111112024-02-06T22:20:21.308Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509101072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111122024-02-06T22:20:21.308Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509144806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111132024-02-06T22:20:21.308Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509179452Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111142024-02-06T22:20:21.308Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509203908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111152024-02-06T22:20:21.308Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509234424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111162024-02-06T22:20:21.308Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509270249Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111172024-02-06T22:20:21.308Z,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111182024-02-06T22:20:21.308Z{"msg":"2b222732-e238-423b-a4d3-ad22be87728d is now active with session: 7903e05d-4f87-4b48-83df-b3324c78742f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509309544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111192024-02-06T22:20:21.308Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.50934135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546,"session_id":"7903e05d-4f87-4b48-83df-b3324c78742f"}
111202024-02-06T22:20:21.308Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:20.509373705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546}
111212024-02-06T22:20:21.308ZLoading write count information from file "/tmp/test_repair_verify.data"
111222024-02-06T22:20:21.308ZRun Repair workload
111232024-02-06T22:20:21.312Z01/30 Write block 448 len 28672 data: 5 5 5 5 4 4 6
111242024-02-06T22:20:21.312Z02/30 Read block 247 len 20480
111252024-02-06T22:20:21.312Z03/30 Read block 50 len 12288
111262024-02-06T22:20:21.312Z04/30 Read block 293 len 16384
111272024-02-06T22:20:21.312Z05/30 Write block 263 len 32768 data: 13 11 11 12 12 11 10 8
111282024-02-06T22:20:21.312Z06/30 Write block 30 len 28672 data: 7 5 3 4 4 3 4
111292024-02-06T22:20:21.312Z07/30 Read block 214 len 20480
111302024-02-06T22:20:21.315Z08/30 Flush
111312024-02-06T22:20:21.315Z09/30 Write block 326 len 16384 data: 5 5 4 5
111322024-02-06T22:20:21.315Z10/30 Read block 539 len 24576
111332024-02-06T22:20:21.318Z11/30 Write block 520 len 8192 data: 5 5
111342024-02-06T22:20:21.318Z12/30 Read block 193 len 32768
111352024-02-06T22:20:21.318Z13/30 Flush
111362024-02-06T22:20:21.321Z14/30 Write block 308 len 28672 data: 8 7 9 10 10 11 11
111372024-02-06T22:20:21.321Z15/30 Read block 16 len 40960
111382024-02-06T22:20:21.326Z16/30 Read block 234 len 4096
111392024-02-06T22:20:21.326Z17/30 Write block 403 len 16384 data: 10 9 10 11
111402024-02-06T22:20:21.326Z18/30 Read block 480 len 20480
111412024-02-06T22:20:21.326Z19/30 Write block 408 len 24576 data: 9 10 10 11 8 7
111422024-02-06T22:20:21.326Z20/30 Read block 583 len 20480
111432024-02-06T22:20:21.331Z21/30 Write block 489 len 20480 data: 6 4 5 6 7
111442024-02-06T22:20:21.331Z22/30 Read block 96 len 24576
111452024-02-06T22:20:21.331Z23/30 Write block 10 len 8192 data: 9 6
111462024-02-06T22:20:21.331Z24/30 Write block 386 len 12288 data: 10 8 7
111472024-02-06T22:20:21.331Z25/30 Read block 50 len 12288
111482024-02-06T22:20:21.331Z26/30 Read block 84 len 24576
111492024-02-06T22:20:21.331Z27/30 Write block 200 len 12288 data: 7 8 11
111502024-02-06T22:20:21.334Z28/30 Write block 71 len 32768 data: 6 7 8 11 10 7 7 6
111512024-02-06T22:20:21.334Z29/30 Write block 386 len 4096 data: 11
111522024-02-06T22:20:21.335Z30/30 Read block 368 len 40960
111532024-02-06T22:20:21.335Z----------------------------------------------------------------
111542024-02-06T22:20:21.335Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
111552024-02-06T22:20:21.335ZStates: Active Active Active
111562024-02-06T22:20:21.335ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
111572024-02-06T22:20:21.335Z 1 Acked 1000 Write 7 Done Done Done false
111582024-02-06T22:20:21.335Z 2 Acked 1001 Read 5 Done Done Done false
111592024-02-06T22:20:21.335Z 3 Acked 1002 Read 3 Done Done Done false
111602024-02-06T22:20:21.335Z 4 Acked 1003 Read 4 Sent Done Done false
111612024-02-06T22:20:21.335Z 5 Acked 1004 Write 8 Sent Done Done false
111622024-02-06T22:20:21.335Z 6 Acked 1005 Write 7 Sent Done Done false
111632024-02-06T22:20:21.335Z 7 Acked 1006 Read 5 Sent Done Done false
111642024-02-06T22:20:21.335Z 8 Acked 1007 Flush 0 Sent Done Done false
111652024-02-06T22:20:21.335Z 9 Acked 1008 Write 4 Sent Done Done false
111662024-02-06T22:20:21.335Z 10 Acked 1009 Read 6 Sent Done Done false
111672024-02-06T22:20:21.335Z 11 Acked 1010 Write 2 Sent Done Done false
111682024-02-06T22:20:21.335Z 12 Acked 1011 Read 8 Sent Done Done false
111692024-02-06T22:20:21.335Z 13 Acked 1012 Flush 0 Sent Done Done false
111702024-02-06T22:20:21.335Z 14 Acked 1013 Write 7 Sent Done Done false
111712024-02-06T22:20:21.335Z 15 Acked 1014 Read 10 Sent Done Done false
111722024-02-06T22:20:21.335Z 16 Acked 1015 Read 1 Sent Done Done false
111732024-02-06T22:20:21.335Z 17 Acked 1016 Write 4 Sent Done Done false
111742024-02-06T22:20:21.335Z 18 Acked 1017 Read 5 Sent Done Done false
111752024-02-06T22:20:21.335Z 19 Acked 1018 Write 6 Sent Done Done false
111762024-02-06T22:20:21.335Z 20 Acked 1019 Read 5 Sent Done Done false
111772024-02-06T22:20:21.336Z 21 Acked 1020 Write 5 Sent Done Done false
111782024-02-06T22:20:21.336Z 22 Acked 1021 Read 6 Sent Done Done false
111792024-02-06T22:20:21.336Z 23 Acked 1022 Write 2 Sent Done Done false
111802024-02-06T22:20:21.336Z 24 Acked 1023 Write 3 Sent Done Done false
111812024-02-06T22:20:21.336Z 25 Acked 1024 Read 3 Sent Done Done false
111822024-02-06T22:20:21.336Z 26 Acked 1025 Read 6 Sent Done Done false
111832024-02-06T22:20:21.336Z 27 Acked 1026 Write 3 Sent Done Done false
111842024-02-06T22:20:21.336Z 28 Acked 1027 Write 8 Sent Done Done false
111852024-02-06T22:20:21.336Z 29 Acked 1028 Write 1 Sent Done Done false
111862024-02-06T22:20:21.336Z 30 Acked 1029 Read 10 Sent Done Sent false
111872024-02-06T22:20:21.336Z STATES DS:0 DS:1 DS:2 TOTAL
111882024-02-06T22:20:21.336Z New 0 0 0 0
111892024-02-06T22:20:21.336Z Sent 27 0 1 28
111902024-02-06T22:20:21.336Z Done 3 30 29 62
111912024-02-06T22:20:21.336Z Skipped 0 0 0 0
111922024-02-06T22:20:21.336Z Error 0 0 0 0
111932024-02-06T22:20:21.336ZLast Flush: 0 1012 1012
111942024-02-06T22:20:21.336ZDownstairs last five completed:
111952024-02-06T22:20:21.336ZUpstairs last five completed: 30 29 28 27 26
111962024-02-06T22:20:21.339Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:20.538813077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1546}
111972024-02-06T22:20:21.339ZWrote out file "/tmp/test_repair_verify.data"
111982024-02-06T22:20:21.340Z
111992024-02-06T22:20:21.343ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112002024-02-06T22:20:21.364ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112012024-02-06T22:20:21.364Z 0 000-019 95 95 95 179 179 179 F T T <---
112022024-02-06T22:20:21.365Z 1 020-039 92 102 102 174 193 193 F F F <---
112032024-02-06T22:20:21.365Z 2 040-059 90 90 90 168 168 168 F F F
112042024-02-06T22:20:21.365Z 3 060-079 99 99 99 188 188 188 F T T <---
112052024-02-06T22:20:21.365Z 4 080-099 97 97 97 185 185 185 F F F
112062024-02-06T22:20:21.365Z 5 100-119 98 98 98 187 187 187 F F F
112072024-02-06T22:20:21.365Z 6 120-139 100 100 100 189 189 189 F F F
112082024-02-06T22:20:21.365Z 7 140-159 100 100 100 191 191 191 F F F
112092024-02-06T22:20:21.365Z 8 160-179 101 101 101 192 192 192 F F F
112102024-02-06T22:20:21.365Z 9 180-199 100 100 100 189 189 189 F F F
112112024-02-06T22:20:21.383Z 10 200-219 101 101 101 192 192 192 F T T <---
112122024-02-06T22:20:21.383Z 11 220-239 94 94 94 177 177 177 F F F
112132024-02-06T22:20:21.384Z 12 240-259 101 101 101 192 192 192 F F F
112142024-02-06T22:20:21.384Z 13 260-279 101 102 102 192 193 193 F F F <---
112152024-02-06T22:20:21.384Z 14 280-299 100 100 100 189 189 189 F F F
112162024-02-06T22:20:21.384Z 15 300-319 100 100 100 191 191 191 F T T <---
112172024-02-06T22:20:21.384Z 16 320-339 100 102 102 190 194 194 F F F <---
112182024-02-06T22:20:21.384Z 17 340-359 100 100 100 189 189 189 F F F
112192024-02-06T22:20:21.384Z 18 360-379 99 99 99 188 188 188 F F F
112202024-02-06T22:20:21.384Z 19 380-399 100 100 100 191 191 191 F T T <---
112212024-02-06T22:20:21.384Z 20 400-419 101 101 101 192 192 192 F T T <---
112222024-02-06T22:20:21.384Z 21 420-439 101 101 101 192 192 192 F F F
112232024-02-06T22:20:21.384Z 22 440-459 101 102 102 192 193 193 T F F <---
112242024-02-06T22:20:21.384Z 23 460-479 101 101 101 192 192 192 F F F
112252024-02-06T22:20:21.384Z 24 480-499 96 96 96 184 184 184 F T T <---
112262024-02-06T22:20:21.385Z 25 500-519 99 99 99 188 188 188 F F F
112272024-02-06T22:20:21.385Z 26 520-539 92 102 102 173 194 194 F F F <---
112282024-02-06T22:20:21.385Z 27 540-559 101 101 101 192 192 192 F F F
112292024-02-06T22:20:21.385Z 28 560-579 101 101 101 192 192 192 F F F
112302024-02-06T22:20:21.385Z 29 580-599 101 101 101 192 192 192 F F F
112312024-02-06T22:20:21.385ZMax gen: 102, Max flush: 194
112322024-02-06T22:20:21.385ZError: Difference in extent metadata found!
112332024-02-06T22:20:21.385ZA Difference in extent metadata is expected here
112342024-02-06T22:20:21.385ZOn loop 51
112352024-02-06T22:20:21.385Z
112362024-02-06T22:20:21.385ZVerifying data now
112372024-02-06T22:20:26.974ZNo change after verify
112382024-02-06T22:20:26.974ZLoop: 51 Downstairs dump after verify (and repair):
112392024-02-06T22:20:26.992ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112402024-02-06T22:20:26.992Z 0 000-019 103 103 103 195 195 195 F F F
112412024-02-06T22:20:26.993Z 1 020-039 102 102 102 193 193 193 F F F
112422024-02-06T22:20:26.993Z 2 040-059 90 90 90 168 168 168 F F F
112432024-02-06T22:20:26.993Z 3 060-079 103 103 103 195 195 195 F F F
112442024-02-06T22:20:26.993Z 4 080-099 97 97 97 185 185 185 F F F
112452024-02-06T22:20:26.993Z 5 100-119 98 98 98 187 187 187 F F F
112462024-02-06T22:20:26.993Z 6 120-139 100 100 100 189 189 189 F F F
112472024-02-06T22:20:27.002Z 7 140-159 100 100 100 191 191 191 F F F
112482024-02-06T22:20:27.002Z 8 160-179 101 101 101 192 192 192 F F F
112492024-02-06T22:20:27.003Z 9 180-199 100 100 100 189 189 189 F F F
112502024-02-06T22:20:27.003Z 10 200-219 103 103 103 195 195 195 F F F
112512024-02-06T22:20:27.003Z 11 220-239 94 94 94 177 177 177 F F F
112522024-02-06T22:20:27.003Z 12 240-259 101 101 101 192 192 192 F F F
112532024-02-06T22:20:27.003Z 13 260-279 102 102 102 193 193 193 F F F
112542024-02-06T22:20:27.003Z 14 280-299 100 100 100 189 189 189 F F F
112552024-02-06T22:20:27.003Z 15 300-319 103 103 103 195 195 195 F F F
112562024-02-06T22:20:27.003Z 16 320-339 102 102 102 194 194 194 F F F
112572024-02-06T22:20:27.003Z 17 340-359 100 100 100 189 189 189 F F F
112582024-02-06T22:20:27.003Z 18 360-379 99 99 99 188 188 188 F F F
112592024-02-06T22:20:27.004Z 19 380-399 103 103 103 195 195 195 F F F
112602024-02-06T22:20:27.004Z 20 400-419 103 103 103 195 195 195 F F F
112612024-02-06T22:20:27.004Z 21 420-439 101 101 101 192 192 192 F F F
112622024-02-06T22:20:27.004Z 22 440-459 102 102 102 193 193 193 F F F
112632024-02-06T22:20:27.004Z 23 460-479 101 101 101 192 192 192 F F F
112642024-02-06T22:20:27.004Z 24 480-499 103 103 103 195 195 195 F F F
112652024-02-06T22:20:27.004Z 25 500-519 99 99 99 188 188 188 F F F
112662024-02-06T22:20:27.004Z 26 520-539 102 102 102 194 194 194 F F F
112672024-02-06T22:20:27.004Z 27 540-559 101 101 101 192 192 192 F F F
112682024-02-06T22:20:27.004Z 28 560-579 101 101 101 192 192 192 F F F
112692024-02-06T22:20:27.004Z 29 580-599 101 101 101 192 192 192 F F F
112702024-02-06T22:20:27.004ZMax gen: 103, Max flush: 195
112712024-02-06T22:20:27.004Z
112722024-02-06T22:20:27.005ZBegin loop 52
112732024-02-06T22:20:27.005ZDownstairs to restart: 1
112742024-02-06T22:20:27.012Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.208513451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554}
112752024-02-06T22:20:27.013Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
112762024-02-06T22:20:27.013Z2024-02-06T22:20:26.208800542Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554}
112772024-02-06T22:20:27.013Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.208851945Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554}
112782024-02-06T22:20:27.013Z{"msg":"Crucible 6f85d87c-9c9f-457d-a857-6a150efe503b has session id: b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.208895939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112792024-02-06T22:20:27.013Z{"msg":"Upstairs opts: Upstairs UUID: 6f85d87c-9c9f-457d-a857-6a150efe503b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.208954651Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112802024-02-06T22:20:27.013Z{"msg":"Crucible stats registered with UUID: 6f85d87c-9c9f-457d-a857-6a150efe503b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209000325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112812024-02-06T22:20:27.013Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209043259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554}
112822024-02-06T22:20:27.013Z{"msg":"6f85d87c-9c9f-457d-a857-6a150efe503b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209087593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112832024-02-06T22:20:27.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209131007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112842024-02-06T22:20:27.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209178261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112852024-02-06T22:20:27.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209211017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112862024-02-06T22:20:27.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209302754Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112872024-02-06T22:20:27.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209378494Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112882024-02-06T22:20:27.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.20940805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112892024-02-06T22:20:27.014Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209452094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112902024-02-06T22:20:27.014Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209494848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112912024-02-06T22:20:27.014Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209549661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112922024-02-06T22:20:27.014Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209597535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112932024-02-06T22:20:27.014Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209636209Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112942024-02-06T22:20:27.014Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:26.209676134Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112952024-02-06T22:20:27.014Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:26.209720088Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112962024-02-06T22:20:27.014Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:26.209766662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112972024-02-06T22:20:27.015Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209795908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112982024-02-06T22:20:27.015Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.209830703Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
112992024-02-06T22:20:27.015Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.20985341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113002024-02-06T22:20:27.015Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.210234009Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113012024-02-06T22:20:27.015Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.210276243Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113022024-02-06T22:20:27.015Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.21029337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113032024-02-06T22:20:27.015Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.210364781Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113042024-02-06T22:20:27.015Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.210840467Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113052024-02-06T22:20:27.015Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.211056577Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113062024-02-06T22:20:27.015Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.211312353Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113072024-02-06T22:20:27.015Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.211346728Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113082024-02-06T22:20:27.015Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.211361766Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113092024-02-06T22:20:27.015Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.211588855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113102024-02-06T22:20:27.016Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.211622131Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113112024-02-06T22:20:27.016Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:26.211637169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113122024-02-06T22:20:28.010Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.211167071Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113132024-02-06T22:20:28.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.211217624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113142024-02-06T22:20:28.023Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.211944685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113152024-02-06T22:20:28.023Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.212005007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113162024-02-06T22:20:28.023Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.21250423Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113172024-02-06T22:20:28.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.212967837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113182024-02-06T22:20:28.023Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213015081Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113192024-02-06T22:20:28.023Z{"msg":"[0]R flush_numbers[0..12]: [195, 193, 168, 195, 185, 187, 189, 191, 192, 189, 195, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213045567Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113202024-02-06T22:20:28.024Z{"msg":"[0]R generation[0..12]: [103, 102, 90, 103, 97, 98, 100, 100, 101, 100, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213084461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113212024-02-06T22:20:28.024Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213114267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113222024-02-06T22:20:28.024Z{"msg":"[1]R flush_numbers[0..12]: [195, 193, 168, 195, 185, 187, 189, 191, 192, 189, 195, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213140204Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113232024-02-06T22:20:28.024Z{"Wait for a query_work_queue command to finish before sending IO
113242024-02-06T22:20:28.024Zmsg":"[1]R generation[0..12]: [103, 102, 90, 103, 97, 98, 100, 100, 101, 100, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213163151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113252024-02-06T22:20:28.024Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213204855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113262024-02-06T22:20:28.024Z{"msg":"[2]R flush_numbers[0..12]: [195, 193, 168, 195, 185, 187, 189, 191, 192, 189, 195, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213234161Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113272024-02-06T22:20:28.024Z{"msg":"[2]R generation[0..12]: [103, 102, 90, 103, 97, 98, 100, 100, 101, 100, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213270166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113282024-02-06T22:20:28.024Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213309981Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113292024-02-06T22:20:28.024Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213350536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113302024-02-06T22:20:28.024Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213383001Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113312024-02-06T22:20:28.024Z{"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213415917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113322024-02-06T22:20:28.024Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213448462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113332024-02-06T22:20:28.024Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213480688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113342024-02-06T22:20:28.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213511964Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113352024-02-06T22:20:28.024ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113362024-02-06T22:20:28.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213546929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113372024-02-06T22:20:28.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213579854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113382024-02-06T22:20:28.024Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.21361266Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113392024-02-06T22:20:28.024Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213629758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113402024-02-06T22:20:28.024Z{"msg":"6f85d87c-9c9f-457d-a857-6a150efe503b is now active with session: b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213658774Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113412024-02-06T22:20:28.024Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213678221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554,"session_id":"b9b6bc7f-4e0c-4a44-97c4-a1b61f3f4bb0"}
113422024-02-06T22:20:28.024Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:27.213705477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554}
113432024-02-06T22:20:28.025ZLoading write count information from file "/tmp/test_repair_verify.data"
113442024-02-06T22:20:28.025ZRun Repair workload
113452024-02-06T22:20:28.025Z01/30 Write block 568 len 28672 data: 12 10 10 14 14 13 14
113462024-02-06T22:20:28.025Z02/30 Flush
113472024-02-06T22:20:28.032Z03/30 Write block 584 len 28672 data: 7 9 7 10 10 12 15
113482024-02-06T22:20:28.033Z04/30 Read block 416 len 20480
113492024-02-06T22:20:28.033Z05/30 Write block 77 len 16384 data: 8 7 5 5
113502024-02-06T22:20:28.033Z06/30 Flush
113512024-02-06T22:20:28.033Z07/30 Write block 161 len 32768 data: 6 7 7 7 5 5 7 9
113522024-02-06T22:20:28.033Z08/30 Write block 265 len 32768 data: 12 13 13 12 11 9 7 8
113532024-02-06T22:20:28.033Z09/30 Write block 446 len 40960 data: 4 4 6 6 6 6 5 5 7 7
113542024-02-06T22:20:28.033Z10/30 Write block 586 len 40960 data: 8 11 11 13 16 13 12 13 12 10
113552024-02-06T22:20:28.033Z11/30 Read block 202 len 28672
113562024-02-06T22:20:28.033Z12/30 Flush
113572024-02-06T22:20:28.033Z13/30 Write block 146 len 12288 data: 10 11 13
113582024-02-06T22:20:28.033Z14/30 Read block 264 len 24576
113592024-02-06T22:20:28.033Z15/30 Read block 367 len 28672
113602024-02-06T22:20:28.050Z16/30 Write block 333 len 8192 data: 7 10
113612024-02-06T22:20:28.050Z17/30 Write block 505 len 8192 data: 10 11
113622024-02-06T22:20:28.050Z18/30 Write block 526 len 32768 data: 7 7 7 9 11 11 9 9
113632024-02-06T22:20:28.050Z19/30 Write block 98 len 20480 data: 7 7 7 4 5
113642024-02-06T22:20:28.050Z20/30 Read block 477 len 8192
113652024-02-06T22:20:28.051Z21/30 Write block 324 len 8192 data: 5 6
113662024-02-06T22:20:28.051Z22/30 Read block 221 len 20480
113672024-02-06T22:20:28.051Z23/30 Read block 414 len 40960
113682024-02-06T22:20:28.051Z24/30 Write block 479 len 24576 data: 9 6 4 6 7 8
113692024-02-06T22:20:28.051Z25/30 Flush
113702024-02-06T22:20:28.051Z26/30 Write block 117 len 16384 data: 9 10 9 9
113712024-02-06T22:20:28.051Z27/30 Read block 109 len 24576
113722024-02-06T22:20:28.051Z28/30 Read block 61 len 32768
113732024-02-06T22:20:28.051Z29/30 Read block 283 len 40960
113742024-02-06T22:20:28.051Z30/30 Write block 48 len 32768 data: 6 10 11 14 14 16 18 17
113752024-02-06T22:20:28.051Z----------------------------------------------------------------
113762024-02-06T22:20:28.051Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
113772024-02-06T22:20:28.051ZStates: Active Active Active
113782024-02-06T22:20:28.051ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
113792024-02-06T22:20:28.051Z 1 Acked 1000 Write 7 Done Sent Done false
113802024-02-06T22:20:28.051Z 2 Acked 1001 Flush 0 Done Sent Done false
113812024-02-06T22:20:28.051Z 3 Acked 1002 Write 7 Done Sent Done false
113822024-02-06T22:20:28.051Z 4 Acked 1003 Read 5 Done Sent Done false
113832024-02-06T22:20:28.051Z 5 Acked 1004 Write 4 Done Sent Done false
113842024-02-06T22:20:28.051Z 6 Acked 1005 Flush 0 Done Sent Done false
113852024-02-06T22:20:28.051Z 7 Acked 1006 Write 8 Done Sent Done false
113862024-02-06T22:20:28.051Z 8 Acked 1007 Write 8 Done Sent Done false
113872024-02-06T22:20:28.051Z 9 Acked 1008 Write 10 Done Sent Done false
113882024-02-06T22:20:28.052Z 10 Acked 1009 Write 10 Done Sent Done false
113892024-02-06T22:20:28.052Z 11 Acked 1010 Read 7 Done Sent Done false
113902024-02-06T22:20:28.052Z 12 Acked 1011 Flush 0 Done Sent Done false
113912024-02-06T22:20:28.052Z 13 Acked 1012 Write 3 Done Sent Done false
113922024-02-06T22:20:28.052Z 14 Acked 1013 Read 6 Done Sent Done false
113932024-02-06T22:20:28.052Z 15 Acked 1014 Read 7 Done Sent Done false
113942024-02-06T22:20:28.052Z 16 Acked 1015 Write 2 Done Sent Done false
113952024-02-06T22:20:28.052Z 17 Acked 1016 Write 2 Done Sent Done false
113962024-02-06T22:20:28.052Z 18 Acked 1017 Write 8 Done Sent Done false
113972024-02-06T22:20:28.052Z 19 Acked 1018 Write 5 Done Sent Done false
113982024-02-06T22:20:28.052Z 20 Acked 1019 Read 2 Done Sent Done false
113992024-02-06T22:20:28.052Z 21 Acked 1020 Write 2 Done Sent Done false
114002024-02-06T22:20:28.052Z 22 Acked 1021 Read 5 Done Sent Done false
114012024-02-06T22:20:28.052Z 23 Acked 1022 Read 10 Done Sent Done false
114022024-02-06T22:20:28.052Z 24 Acked 1023 Write 6 Done Sent Done false
114032024-02-06T22:20:28.052Z 25 Acked 1024 Flush 0 Done Sent Done false
114042024-02-06T22:20:28.052Z 26 Acked 1025 Write 4 Done Sent Done false
114052024-02-06T22:20:28.052Z 27 Acked 1026 Read 6 Done Sent Done false
114062024-02-06T22:20:28.052Z 28 Acked 1027 Read 8 Done Sent Done false
114072024-02-06T22:20:28.052Z 29 Acked 1028 Read 10 Done Sent Done false
114082024-02-06T22:20:28.052Z 30 Acked 1029 Write 8 Sent Sent Sent false
114092024-02-06T22:20:28.052Z STATES DS:0 DS:1 DS:2 TOTAL
114102024-02-06T22:20:28.052Z New 0 0 0 0
114112024-02-06T22:20:28.052Z Sent 1 30 1 32
114122024-02-06T22:20:28.053Z Done 29 0 29 58
114132024-02-06T22:20:28.053Z Skipped 0 0 0 0
114142024-02-06T22:20:28.053Z Error 0 0 0 0
114152024-02-06T22:20:28.053ZLast Flush: 1024 0 1024
114162024-02-06T22:20:28.053ZDownstairs last five completed:
114172024-02-06T22:20:28.053ZUpstairs last five completed: 30 29 28 27 26
114182024-02-06T22:20:28.053Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:27.246542411Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1554}
114192024-02-06T22:20:28.060ZWrote out file "/tmp/test_repair_verify.data"
114202024-02-06T22:20:28.060Z
114212024-02-06T22:20:28.060ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114222024-02-06T22:20:28.070ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114232024-02-06T22:20:28.070Z 0 000-019 103 103 103 195 195 195 F F F
114242024-02-06T22:20:28.070Z 1 020-039 102 102 102 193 193 193 F F F
114252024-02-06T22:20:28.070Z 2 040-059 90 90 90 168 168 168 T F T <---
114262024-02-06T22:20:28.070Z 3 060-079 104 103 104 197 195 197 F F F <---
114272024-02-06T22:20:28.070Z 4 080-099 104 97 104 199 185 199 F F F <---
114282024-02-06T22:20:28.070Z 5 100-119 104 98 104 199 187 199 T F T <---
114292024-02-06T22:20:28.070Z 6 120-139 100 100 100 189 189 189 T F T <---
114302024-02-06T22:20:28.070Z 7 140-159 104 100 104 199 191 199 F F F <---
114312024-02-06T22:20:28.070Z 8 160-179 104 101 104 198 192 198 F F F <---
114322024-02-06T22:20:28.070Z 9 180-199 100 100 100 189 189 189 F F F
114332024-02-06T22:20:28.070Z 10 200-219 103 103 103 195 195 195 F F F
114342024-02-06T22:20:28.070Z 11 220-239 94 94 94 177 177 177 F F F
114352024-02-06T22:20:28.070Z 12 240-259 101 101 101 192 192 192 F F F
114362024-02-06T22:20:28.070Z 13 260-279 104 102 104 198 193 198 F F F <---
114372024-02-06T22:20:28.074Z 14 280-299 100 100 100 189 189 189 F F F
114382024-02-06T22:20:28.074Z 15 300-319 103 103 103 195 195 195 F F F
114392024-02-06T22:20:28.074Z 16 320-339 104 102 104 199 194 199 F F F <---
114402024-02-06T22:20:28.074Z 17 340-359 100 100 100 189 189 189 F F F
114412024-02-06T22:20:28.074Z 18 360-379 99 99 99 188 188 188 F F F
114422024-02-06T22:20:28.074Z 19 380-399 103 103 103 195 195 195 F F F
114432024-02-06T22:20:28.074Z 20 400-419 103 103 103 195 195 195 F F F
114442024-02-06T22:20:28.074Z 21 420-439 101 101 101 192 192 192 F F F
114452024-02-06T22:20:28.074Z 22 440-459 104 102 104 198 193 198 F F F <---
114462024-02-06T22:20:28.074Z 23 460-479 104 101 104 199 192 199 F F F <---
114472024-02-06T22:20:28.074Z 24 480-499 104 103 104 199 195 199 F F F <---
114482024-02-06T22:20:28.074Z 25 500-519 104 99 104 199 188 199 F F F <---
114492024-02-06T22:20:28.074Z 26 520-539 104 102 104 199 194 199 F F F <---
114502024-02-06T22:20:28.074Z 27 540-559 101 101 101 192 192 192 F F F
114512024-02-06T22:20:28.074Z 28 560-579 104 101 104 196 192 196 F F F <---
114522024-02-06T22:20:28.074Z 29 580-599 104 101 104 198 192 198 F F F <---
114532024-02-06T22:20:28.074ZMax gen: 104, Max flush: 199
114542024-02-06T22:20:28.074ZError: Difference in extent metadata found!
114552024-02-06T22:20:28.074ZA Difference in extent metadata is expected here
114562024-02-06T22:20:28.074ZOn loop 52
114572024-02-06T22:20:28.074Z
114582024-02-06T22:20:28.074ZVerifying data now
114592024-02-06T22:20:32.820ZNo change after verify
114602024-02-06T22:20:32.820ZLoop: 52 Downstairs dump after verify (and repair):
114612024-02-06T22:20:32.835ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114622024-02-06T22:20:32.835Z 0 000-019 103 103 103 195 195 195 F F F
114632024-02-06T22:20:32.835Z 1 020-039 102 102 102 193 193 193 F F F
114642024-02-06T22:20:32.835Z 2 040-059 105 105 105 200 200 200 F F F
114652024-02-06T22:20:32.835Z 3 060-079 104 104 104 197 197 197 F F F
114662024-02-06T22:20:32.835Z 4 080-099 104 104 104 199 199 199 F F F
114672024-02-06T22:20:32.835Z 5 100-119 105 105 105 200 200 200 F F F
114682024-02-06T22:20:32.835Z 6 120-139 105 105 105 200 200 200 F F F
114692024-02-06T22:20:32.835Z 7 140-159 104 104 104 199 199 199 F F F
114702024-02-06T22:20:32.835Z 8 160-179 104 104 104 198 198 198 F F F
114712024-02-06T22:20:32.845Z 9 180-199 100 100 100 189 189 189 F F F
114722024-02-06T22:20:32.845Z 10 200-219 103 103 103 195 195 195 F F F
114732024-02-06T22:20:32.845Z 11 220-239 94 94 94 177 177 177 F F F
114742024-02-06T22:20:32.846Z 12 240-259 101 101 101 192 192 192 F F F
114752024-02-06T22:20:32.846Z 13 260-279 104 104 104 198 198 198 F F F
114762024-02-06T22:20:32.846Z 14 280-299 100 100 100 189 189 189 F F F
114772024-02-06T22:20:32.846Z 15 300-319 103 103 103 195 195 195 F F F
114782024-02-06T22:20:32.846Z 16 320-339 104 104 104 199 199 199 F F F
114792024-02-06T22:20:32.846Z 17 340-359 100 100 100 189 189 189 F F F
114802024-02-06T22:20:32.846Z 18 360-379 99 99 99 188 188 188 F F F
114812024-02-06T22:20:32.846Z 19 380-399 103 103 103 195 195 195 F F F
114822024-02-06T22:20:32.846Z 20 400-419 103 103 103 195 195 195 F F F
114832024-02-06T22:20:32.846Z 21 420-439 101 101 101 192 192 192 F F F
114842024-02-06T22:20:32.846Z 22 440-459 104 104 104 198 198 198 F F F
114852024-02-06T22:20:32.846Z 23 460-479 104 104 104 199 199 199 F F F
114862024-02-06T22:20:32.846Z 24 480-499 104 104 104 199 199 199 F F F
114872024-02-06T22:20:32.846Z 25 500-519 104 104 104 199 199 199 F F F
114882024-02-06T22:20:32.846Z 26 520-539 104 104 104 199 199 199 F F F
114892024-02-06T22:20:32.846Z 27 540-559 101 101 101 192 192 192 F F F
114902024-02-06T22:20:32.846Z 28 560-579 104 104 104 196 196 196 F F F
114912024-02-06T22:20:32.847Z 29 580-599 104 104 104 198 198 198 F F F
114922024-02-06T22:20:32.847ZMax gen: 105, Max flush: 200
114932024-02-06T22:20:32.847Z
114942024-02-06T22:20:32.847ZBegin loop 53
114952024-02-06T22:20:32.847ZDownstairs to restart: 1
114962024-02-06T22:20:32.862Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058231583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562}
114972024-02-06T22:20:32.862Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
114982024-02-06T22:20:32.862Z rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058520314Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562}
114992024-02-06T22:20:32.895Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058562908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562}
115002024-02-06T22:20:32.895Z{"msg":"Crucible 941fc879-1682-44a6-abff-bc0f18672892 has session id: c7c9e7c7-24e4-47b9-834d-8c677e4187e5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058613461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115012024-02-06T22:20:32.895Z{"msg":"Upstairs opts: Upstairs UUID: 941fc879-1682-44a6-abff-bc0f18672892, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058659205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115022024-02-06T22:20:32.895Z{"msg":"Crucible stats registered with UUID: 941fc879-1682-44a6-abff-bc0f18672892","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058704069Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115032024-02-06T22:20:32.896Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058754222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562}
115042024-02-06T22:20:32.896Z{"msg":"941fc879-1682-44a6-abff-bc0f18672892 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058788138Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115052024-02-06T22:20:32.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058812064Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115062024-02-06T22:20:32.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058857018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115072024-02-06T22:20:32.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058883314Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115082024-02-06T22:20:32.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.05891856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115092024-02-06T22:20:32.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.058950336Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115102024-02-06T22:20:32.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059017436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115112024-02-06T22:20:32.896Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059051282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115122024-02-06T22:20:32.896Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059096526Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115132024-02-06T22:20:32.896Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.05913909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115142024-02-06T22:20:32.896Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059179685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115152024-02-06T22:20:32.896Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.05920931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115162024-02-06T22:20:32.896Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059247005Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115172024-02-06T22:20:32.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059867062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115182024-02-06T22:20:32.896Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059911975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115192024-02-06T22:20:32.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.059936612Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115202024-02-06T22:20:32.896Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.060015242Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115212024-02-06T22:20:32.896Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.060554899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115222024-02-06T22:20:32.896Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.060596003Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115232024-02-06T22:20:32.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.060863727Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115242024-02-06T22:20:32.896Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.06091481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115252024-02-06T22:20:32.896Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061109734Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115262024-02-06T22:20:32.896Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061147968Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115272024-02-06T22:20:32.896Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061164946Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115282024-02-06T22:20:32.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061193872Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115292024-02-06T22:20:32.897Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061230177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115302024-02-06T22:20:32.897Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061246885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115312024-02-06T22:20:32.897Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061452387Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115322024-02-06T22:20:32.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.06187594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115332024-02-06T22:20:32.897Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061917924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115342024-02-06T22:20:32.897Z{"msg":"[0]R flush_numbers[0..12]: [195, 193, 200, 197, 199, 200, 200, 199, 198, 189, 195, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.06195244Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115352024-02-06T22:20:32.897Z{"msg":"[0]R generation[0..12]: [103, 102, 105, 104, 104, 105, 105, 104, 104, 100, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061972657Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115362024-02-06T22:20:32.897Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.061999513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115372024-02-06T22:20:32.897Z{"msg":"[1]R flush_numbers[0..12]: [195, 193, 200, 197, 199, 200, 200, 199, 198, 189, 195, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062017831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115382024-02-06T22:20:32.897Z{"msg":"[1]R generation[0..12]: [103, 102, 105, 104, 104, 105, 105, 104, 104, 100, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062035229Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115392024-02-06T22:20:32.897Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062061495Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115402024-02-06T22:20:32.897Z{"msg":"[2]R flush_numbers[0..12]: [195, 193, 200, 197, 199, 200, 200, 199, 198, 189, 195, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062079683Z","hostname":"Wait for a query_work_queue command to finish before sending IO
115412024-02-06T22:20:32.897Zip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115422024-02-06T22:20:32.897Z{"msg":"[2]R generation[0..12]: [103, 102, 105, 104, 104, 105, 105, 104, 104, 100, 103, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.06210175Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115432024-02-06T22:20:32.897Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062119877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115442024-02-06T22:20:32.897Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062151893Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115452024-02-06T22:20:32.897Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.06216955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115462024-02-06T22:20:32.897Z{"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062194627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115472024-02-06T22:20:32.897Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062211405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115482024-02-06T22:20:32.897Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062228143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115492024-02-06T22:20:32.897Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062252229Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115502024-02-06T22:20:32.897Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062269077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115512024-02-06T22:20:32.897Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062287405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115522024-02-06T22:20:32.898Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062304692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115532024-02-06T22:20:32.898Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062329039Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115542024-02-06T22:20:32.898Z{"msg":"941fc879-1682-44a6-abff-bc0f18672892 is now active with session: c7c9e7c7-24e4-47b9-834d-8c677e4187e5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062346167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115552024-02-06T22:20:32.898Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062361904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562,"session_id":"c7c9e7c7-24e4-47b9-834d-8c677e4187e5"}
115562024-02-06T22:20:32.898Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:32.062377153Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562}
115572024-02-06T22:20:32.898ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115582024-02-06T22:20:32.898ZLoading write count information from file "/tmp/test_repair_verify.data"
115592024-02-06T22:20:32.898ZRun Repair workload
115602024-02-06T22:20:32.898Z01/30 Write block 586 len 24576 data: 9 12 12 14 17 14
115612024-02-06T22:20:32.898Z02/30 Read block 78 len 32768
115622024-02-06T22:20:32.898Z03/30 Write block 199 len 24576 data: 8 8 9 12 10 11
115632024-02-06T22:20:32.898Z04/30 Read block 481 len 8192
115642024-02-06T22:20:32.898Z05/30 Write block 76 len 24576 data: 8 9 8 6 6 6
115652024-02-06T22:20:32.898Z06/30 Read block 425 len 20480
115662024-02-06T22:20:32.898Z07/30 Read block 258 len 12288
115672024-02-06T22:20:32.898Z08/30 Write block 540 len 8192 data: 4 6
115682024-02-06T22:20:32.902Z09/30 Write block 388 len 12288 data: 8 7 7
115692024-02-06T22:20:32.902Z10/30 Write block 174 len 20480 data: 13 12 10 9 10
115702024-02-06T22:20:32.902Z11/30 Read block 181 len 28672
115712024-02-06T22:20:32.902Z12/30 Write block 166 len 24576 data: 6 8 10 12 11 8
115722024-02-06T22:20:32.903Z13/30 Write block 214 len 4096 data: 11
115732024-02-06T22:20:32.903Z14/30 Read block 427 len 12288
115742024-02-06T22:20:32.903Z15/30 Read block 109 len 28672
115752024-02-06T22:20:32.903Z16/30 Read block 159 len 32768
115762024-02-06T22:20:32.903Z17/30 Read block 496 len 36864
115772024-02-06T22:20:32.903Z18/30 Write block 110 len 36864 data: 8 6 8 9 11 11 8 10 11
115782024-02-06T22:20:32.903Z19/30 Read block 38 len 36864
115792024-02-06T22:20:32.903Z20/30 Write block 542 len 28672 data: 6 8 10 8 9 7 6
115802024-02-06T22:20:32.903Z21/30 Write block 492 len 32768 data: 7 8 8 8 8 8 9 9
115812024-02-06T22:20:32.903Z22/30 Read block 228 len 20480
115822024-02-06T22:20:32.903Z23/30 Write block 130 len 28672 data: 13 12 9 9 8 7 6
115832024-02-06T22:20:32.903Z24/30 Flush
115842024-02-06T22:20:32.903Z25/30 Write block 263 len 36864 data: 14 12 13 14 14 13 12 10 8
115852024-02-06T22:20:32.903Z26/30 Read block 113 len 16384
115862024-02-06T22:20:32.903Z27/30 Read block 378 len 36864
115872024-02-06T22:20:32.903Z28/30 Read block 555 len 28672
115882024-02-06T22:20:32.903Z29/30 Read block 205 len 16384
115892024-02-06T22:20:32.903Z30/30 Read block 29 len 4096
115902024-02-06T22:20:32.903Z----------------------------------------------------------------
115912024-02-06T22:20:32.903Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
115922024-02-06T22:20:32.903ZStates: Active Active Active
115932024-02-06T22:20:32.903ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115942024-02-06T22:20:32.903Z 1 Acked 1000 Write 6 Done Sent Done false
115952024-02-06T22:20:32.903Z 2 Acked 1001 Read 8 Done Sent Done false
115962024-02-06T22:20:32.904Z 3 Acked 1002 Write 6 Done Sent Done false
115972024-02-06T22:20:32.904Z 4 Acked 1003 Read 2 Done Sent Done false
115982024-02-06T22:20:32.904Z 5 Acked 1004 Write 6 Done Sent Done false
115992024-02-06T22:20:32.904Z 6 Acked 1005 Read 5 Done Sent Done false
116002024-02-06T22:20:32.904Z 7 Acked 1006 Read 3 Done Sent Done false
116012024-02-06T22:20:32.904Z 8 Acked 1007 Write 2 Done Sent Done false
116022024-02-06T22:20:32.904Z 9 Acked 1008 Write 3 Done Sent Done false
116032024-02-06T22:20:32.904Z 10 Acked 1009 Write 5 Done Sent Done false
116042024-02-06T22:20:32.904Z 11 Acked 1010 Read 7 Done Sent Done false
116052024-02-06T22:20:32.904Z 12 Acked 1011 Write 6 Done Sent Done false
116062024-02-06T22:20:32.904Z 13 Acked 1012 Write 1 Done Sent Done false
116072024-02-06T22:20:32.904Z 14 Acked 1013 Read 3 Done Sent Done false
116082024-02-06T22:20:32.904Z 15 Acked 1014 Read 7 Done Sent Done false
116092024-02-06T22:20:32.904Z 16 Acked 1015 Read 8 Done Sent Done false
116102024-02-06T22:20:32.904Z 17 Acked 1016 Read 9 Done Sent Done false
116112024-02-06T22:20:32.904Z 18 Acked 1017 Write 9 Done Sent Done false
116122024-02-06T22:20:32.904Z 19 Acked 1018 Read 9 Done Sent Done false
116132024-02-06T22:20:32.904Z 20 Acked 1019 Write 7 Done Sent Done false
116142024-02-06T22:20:32.904Z 21 Acked 1020 Write 8 Done Sent Done false
116152024-02-06T22:20:32.904Z 22 Acked 1021 Read 5 Done Sent Done false
116162024-02-06T22:20:32.904Z 23 Acked 1022 Write 7 Done Sent Done false
116172024-02-06T22:20:32.904Z 24 Acked 1023 Flush 0 Done Sent Done false
116182024-02-06T22:20:32.904Z 25 Acked 1024 Write 9 Done Sent Done false
116192024-02-06T22:20:32.904Z 26 Acked 1025 Read 4 Done Sent Done false
116202024-02-06T22:20:32.904Z 27 Acked 1026 Read 9 Done Sent Done false
116212024-02-06T22:20:32.905Z 28 Acked 1027 Read 7 Done Sent Done false
116222024-02-06T22:20:32.905Z 29 Acked 1028 Read 4 Done Sent Done false
116232024-02-06T22:20:32.905Z 30 Acked 1029 Read 1 Done Sent Sent false
116242024-02-06T22:20:32.905Z STATES DS:0 DS:1 DS:2 TOTAL
116252024-02-06T22:20:32.905Z New 0 0 0 0
116262024-02-06T22:20:32.905Z Sent 0 30 1 31
116272024-02-06T22:20:32.905Z Done 30 0 29 59
116282024-02-06T22:20:32.905Z Skipped 0 0 0 0
116292024-02-06T22:20:32.905Z Error 0 0 0 0
116302024-02-06T22:20:32.905ZLast Flush: 1023 0 1023
116312024-02-06T22:20:32.905ZDownstairs last five completed:
116322024-02-06T22:20:32.905ZUpstairs last five completed: 30 29 28 27 26
116332024-02-06T22:20:32.905Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:32.093738675Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1562}
116342024-02-06T22:20:32.905ZWrote out file "/tmp/test_repair_verify.data"
116352024-02-06T22:20:32.905Z
116362024-02-06T22:20:32.905ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
116372024-02-06T22:20:32.913ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116382024-02-06T22:20:32.913Z 0 000-019 103 103 103 195 195 195 F F F
116392024-02-06T22:20:32.913Z 1 020-039 102 102 102 193 193 193 F F F
116402024-02-06T22:20:32.913Z 2 040-059 105 105 105 200 200 200 F F F
116412024-02-06T22:20:32.916Z 3 060-079 106 104 106 201 197 201 F F F <---
116422024-02-06T22:20:32.916Z 4 080-099 106 104 106 201 199 201 F F F <---
116432024-02-06T22:20:32.916Z 5 100-119 106 105 106 201 200 201 F F F <---
116442024-02-06T22:20:32.916Z 6 120-139 106 105 106 201 200 201 F F F <---
116452024-02-06T22:20:32.916Z 7 140-159 104 104 104 199 199 199 F F F
116462024-02-06T22:20:32.916Z 8 160-179 106 104 106 201 198 201 F F F <---
116472024-02-06T22:20:32.917Z 9 180-199 106 100 106 201 189 201 F F F <---
116482024-02-06T22:20:32.917Z 10 200-219 106 103 106 201 195 201 F F F <---
116492024-02-06T22:20:32.917Z 11 220-239 94 94 94 177 177 177 F F F
116502024-02-06T22:20:32.917Z 12 240-259 101 101 101 192 192 192 F F F
116512024-02-06T22:20:32.917Z 13 260-279 104 104 104 198 198 198 T F T <---
116522024-02-06T22:20:32.917Z 14 280-299 100 100 100 189 189 189 F F F
116532024-02-06T22:20:32.917Z 15 300-319 103 103 103 195 195 195 F F F
116542024-02-06T22:20:32.917Z 16 320-339 104 104 104 199 199 199 F F F
116552024-02-06T22:20:32.917Z 17 340-359 100 100 100 189 189 189 F F F
116562024-02-06T22:20:32.917Z 18 360-379 99 99 99 188 188 188 F F F
116572024-02-06T22:20:32.917Z 19 380-399 106 103 106 201 195 201 F F F <---
116582024-02-06T22:20:32.917Z 20 400-419 103 103 103 195 195 195 F F F
116592024-02-06T22:20:32.917Z 21 420-439 101 101 101 192 192 192 F F F
116602024-02-06T22:20:32.917Z 22 440-459 104 104 104 198 198 198 F F F
116612024-02-06T22:20:32.917Z 23 460-479 104 104 104 199 199 199 F F F
116622024-02-06T22:20:32.917Z 24 480-499 106 104 106 201 199 201 F F F <---
116632024-02-06T22:20:32.917Z 25 500-519 104 104 104 199 199 199 F F F
116642024-02-06T22:20:32.917Z 26 520-539 104 104 104 199 199 199 F F F
116652024-02-06T22:20:32.917Z 27 540-559 106 101 106 201 192 201 F F F <---
116662024-02-06T22:20:32.917Z 28 560-579 104 104 104 196 196 196 F F F
116672024-02-06T22:20:32.917Z 29 580-599 106 104 106 201 198 201 F F F <---
116682024-02-06T22:20:32.917ZMax gen: 106, Max flush: 201
116692024-02-06T22:20:32.917ZError: Difference in extent metadata found!
116702024-02-06T22:20:32.917ZA Difference in extent metadata is expected here
116712024-02-06T22:20:32.918ZOn loop 53
116722024-02-06T22:20:32.918Z
116732024-02-06T22:20:33.005ZVerifying data now
116742024-02-06T22:20:37.931ZNo change after verify
116752024-02-06T22:20:37.931ZLoop: 53 Downstairs dump after verify (and repair):
116762024-02-06T22:20:37.942ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116772024-02-06T22:20:37.942Z 0 000-019 103 103 103 195 195 195 F F F
116782024-02-06T22:20:37.942Z 1 020-039 102 102 102 193 193 193 F F F
116792024-02-06T22:20:37.942Z 2 040-059 105 105 105 200 200 200 F F F
116802024-02-06T22:20:37.942Z 3 060-079 106 106 106 201 201 201 F F F
116812024-02-06T22:20:37.942Z 4 080-099 106 106 106 201 201 201 F F F
116822024-02-06T22:20:37.942Z 5 100-119 106 106 106 201 201 201 F F F
116832024-02-06T22:20:37.942Z 6 120-139 106 106 106 201 201 201 F F F
116842024-02-06T22:20:37.942Z 7 140-159 104 104 104 199 199 199 F F F
116852024-02-06T22:20:37.942Z 8 160-179 106 106 106 201 201 201 F F F
116862024-02-06T22:20:37.942Z 9 180-199 106 106 106 201 201 201 F F F
116872024-02-06T22:20:37.942Z 10 200-219 106 106 106 201 201 201 F F F
116882024-02-06T22:20:37.942Z 11 220-239 94 94 94 177 177 177 F F F
116892024-02-06T22:20:37.942Z 12 240-259 101 101 101 192 192 192 F F F
116902024-02-06T22:20:37.942Z 13 260-279 107 107 107 202 202 202 F F F
116912024-02-06T22:20:37.942Z 14 280-299 100 100 100 189 189 189 F F F
116922024-02-06T22:20:37.943Z 15 300-319 103 103 103 195 195 195 F F F
116932024-02-06T22:20:37.943Z 16 320-339 104 104 104 199 199 199 F F F
116942024-02-06T22:20:37.943Z 17 340-359 100 100 100 189 189 189 F F F
116952024-02-06T22:20:37.943Z 18 360-379 99 99 99 188 188 188 F F F
116962024-02-06T22:20:37.943Z 19 380-399 106 106 106 201 201 201 F F F
116972024-02-06T22:20:37.943Z 20 400-419 103 103 103 195 195 195 F F F
116982024-02-06T22:20:37.943Z 21 420-439 101 101 101 192 192 192 F F F
116992024-02-06T22:20:37.943Z 22 440-459 104 104 104 198 198 198 F F F
117002024-02-06T22:20:37.943Z 23 460-479 104 104 104 199 199 199 F F F
117012024-02-06T22:20:37.943Z 24 480-499 106 106 106 201 201 201 F F F
117022024-02-06T22:20:37.943Z 25 500-519 104 104 104 199 199 199 F F F
117032024-02-06T22:20:37.943Z 26 520-539 104 104 104 199 199 199 F F F
117042024-02-06T22:20:37.943Z 27 540-559 106 106 106 201 201 201 F F F
117052024-02-06T22:20:37.943Z 28 560-579 104 104 104 196 196 196 F F F
117062024-02-06T22:20:37.943Z 29 580-599 106 106 106 201 201 201 F F F
117072024-02-06T22:20:37.943ZMax gen: 107, Max flush: 202
117082024-02-06T22:20:37.943Z
117092024-02-06T22:20:37.943ZBegin loop 54
117102024-02-06T22:20:37.943ZDownstairs to restart: 1
117112024-02-06T22:20:37.943Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750369147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570}
117122024-02-06T22:20:37.943Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750622233Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570}
117132024-02-06T22:20:37.943Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750650309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570}
117142024-02-06T22:20:37.943Z{"msg":"Crucible 74a6d831-2244-46ba-aee2-c806a85ed438 has session id: d5949b88-3abd-4c68-8bf8-7f6365315c72","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750685834Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117152024-02-06T22:20:37.943ZCrucible runtime is spawned
117162024-02-06T22:20:37.943Z{"msg":"Upstairs opts: Upstairs UUID: 74a6d831-2244-46ba-aee2-c806a85ed438, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750752365Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117172024-02-06T22:20:37.944Z{"msg":"Crucible stats registered with UUID: 74a6d831-2244-46ba-aee2-c806a85ed438","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.75078985Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117182024-02-06T22:20:37.944Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750826655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570}
117192024-02-06T22:20:37.944Z{"msg":"74a6d831-2244-46ba-aee2-c806a85ed438 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750905045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117202024-02-06T22:20:37.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.75093908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117212024-02-06T22:20:37.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.750965796Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117222024-02-06T22:20:37.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751004131Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117232024-02-06T22:20:37.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751030738Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117242024-02-06T22:20:37.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751058634Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117252024-02-06T22:20:37.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.7510862Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117262024-02-06T22:20:37.944Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751113147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117272024-02-06T22:20:37.944Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751148312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117282024-02-06T22:20:37.944Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751183827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117292024-02-06T22:20:37.944Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:36.751230161Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117302024-02-06T22:20:37.944Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:36.751273815Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117312024-02-06T22:20:37.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751299891Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117322024-02-06T22:20:37.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751333627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117332024-02-06T22:20:37.944Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:36.751363433Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117342024-02-06T22:20:37.944Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751389239Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117352024-02-06T22:20:37.944Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751413626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117362024-02-06T22:20:37.944Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751523111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117372024-02-06T22:20:37.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.751967771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117382024-02-06T22:20:37.944Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.752040721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117392024-02-06T22:20:37.945Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.752073397Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117402024-02-06T22:20:37.945Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.752111792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117412024-02-06T22:20:37.945Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.752594427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117422024-02-06T22:20:37.945Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.752775652Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117432024-02-06T22:20:37.945Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.753147222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117442024-02-06T22:20:37.945Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.753205994Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117452024-02-06T22:20:37.945Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.753225461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117462024-02-06T22:20:37.945Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.753239619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117472024-02-06T22:20:37.945Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.753253837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117482024-02-06T22:20:37.945Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:36.753267665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117492024-02-06T22:20:38.550Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.751814779Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117502024-02-06T22:20:38.556Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.752024751Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117512024-02-06T22:20:38.556Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.752754602Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117522024-02-06T22:20:38.556Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.752801186Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117532024-02-06T22:20:38.556Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753275922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117542024-02-06T22:20:38.556Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753749758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117552024-02-06T22:20:38.556Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753791792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117562024-02-06T22:20:38.556Z{"msg":"[0]R flush_numbers[0..12]: [195, 193, 200, 201, 201, 201, 201, 199, 201, 201, 201, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753829647Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117572024-02-06T22:20:38.556Z{"msg":"[0]R generation[0..12]: [103, 102, 105, 106, 106, 106, 106, 104, 106, 106, 106, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753867492Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117582024-02-06T22:20:38.556Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753895858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117592024-02-06T22:20:38.556Z{"msg":"[1]R flush_numbers[0..12]: [195, 193, 200, 201, 201, 201, 201, 199, 201, 201, 201, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753915256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117602024-02-06T22:20:38.557ZWait for a query_work_queue command to finish before sending IO
117612024-02-06T22:20:38.557Z{"msg":"[1]R generation[0..12]: [103, 102, 105, 106, 106, 106, 106, 104, 106, 106, 106, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753943842Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117622024-02-06T22:20:38.557Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753963759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117632024-02-06T22:20:38.557Z{"msg":"[2]R flush_numbers[0..12]: [195, 193, 200, 201, 201, 201, 201, 199, 201, 201, 201, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.753982666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117642024-02-06T22:20:38.557Z{"msg":"[2]R generation[0..12]: [103, 102, 105, 106, 106, 106, 106, 104, 106, 106, 106, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754018671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117652024-02-06T22:20:38.557Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754035889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117662024-02-06T22:20:38.557Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754049547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117672024-02-06T22:20:38.557Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754071524Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117682024-02-06T22:20:38.557Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754087032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117692024-02-06T22:20:38.557Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754112079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117702024-02-06T22:20:38.557Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754127147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117712024-02-06T22:20:38.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754155803Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117722024-02-06T22:20:38.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.7541801Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117732024-02-06T22:20:38.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754209986Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117742024-02-06T22:20:38.557Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754234532Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117752024-02-06T22:20:38.557Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754261269Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117762024-02-06T22:20:38.557Z{"msg":"74a6d831-2244-46ba-aee2-c806a85ed438 is now active with session: d5949b88-3abd-4c68-8bf8-7f6365315c72","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754285046Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117772024-02-06T22:20:38.557Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754310532Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570,"session_id":"d5949b88-3abd-4c68-8bf8-7f6365315c72"}
117782024-02-06T22:20:38.558Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:37.754332619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570}
117792024-02-06T22:20:38.558ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
117802024-02-06T22:20:38.558ZLoading write count information from file "/tmp/test_repair_verify.data"
117812024-02-06T22:20:38.558ZRun Repair workload
117822024-02-06T22:20:38.558Z01/30 Write block 479 len 16384 data: 10 7 5 7
117832024-02-06T22:20:38.561Z02/30 Flush
117842024-02-06T22:20:38.561Z03/30 Write block 45 len 40960 data: 6 7 7 7 11 12 15 15 17 19
117852024-02-06T22:20:38.561Z04/30 Write block 65 len 32768 data: 6 7 7 8 8 8 7 8
117862024-02-06T22:20:38.562Z05/30 Write block 376 len 40960 data: 10 13 12 10 10 11 9 9 8 10
117872024-02-06T22:20:38.562Z06/30 Read block 496 len 40960
117882024-02-06T22:20:38.565Z07/30 Write block 83 len 28672 data: 9 8 8 8 8 7 8
117892024-02-06T22:20:38.565Z08/30 Write block 295 len 32768 data: 13 11 8 7 8 10 11 10
117902024-02-06T22:20:38.565Z09/30 Read block 305 len 32768
117912024-02-06T22:20:38.565Z10/30 Read block 208 len 36864
117922024-02-06T22:20:38.568Z11/30 Write block 40 len 28672 data: 11 9 8 8 7 7 8
117932024-02-06T22:20:38.568Z12/30 Write block 483 len 24576 data: 8 9 7 8 8 7
117942024-02-06T22:20:38.569Z13/30 Read block 235 len 24576
117952024-02-06T22:20:38.572Z14/30 Write block 13 len 24576 data: 5 7 8 7 6 6
117962024-02-06T22:20:38.572Z15/30 Write block 439 len 16384 data: 9 7 5 7
117972024-02-06T22:20:38.572Z16/30 Write block 440 len 12288 data: 8 6 8
117982024-02-06T22:20:38.572Z17/30 Read block 383 len 20480
117992024-02-06T22:20:38.572Z18/30 Flush
118002024-02-06T22:20:38.577Z19/30 Write block 187 len 12288 data: 13 9 8
118012024-02-06T22:20:38.580Z20/30 Read block 453 len 24576
118022024-02-06T22:20:38.580Z21/30 Read block 6 len 36864
118032024-02-06T22:20:38.580Z22/30 Write block 104 len 24576 data: 4 6 6 8 11 7
118042024-02-06T22:20:38.580Z23/30 Read block 78 len 36864
118052024-02-06T22:20:38.583Z24/30 Read block 323 len 4096
118062024-02-06T22:20:38.584Z25/30 Read block 519 len 28672
118072024-02-06T22:20:38.584Z26/30 Read block 344 len 32768
118082024-02-06T22:20:38.587Z27/30 Read block 485 len 4096
118092024-02-06T22:20:38.587Z28/30 Write block 149 len 36864 data: 14 11 9 9 12 11 12 10 12
118102024-02-06T22:20:38.588Z29/30 Write block 331 len 32768 data: 5 5 8 11 10 11 10 11
118112024-02-06T22:20:38.588Z30/30 Write block 75 len 16384 data: 11 9 10 9
118122024-02-06T22:20:38.588Z----------------------------------------------------------------
118132024-02-06T22:20:38.588Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
118142024-02-06T22:20:38.588ZStates: Active Active Active
118152024-02-06T22:20:38.588ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118162024-02-06T22:20:38.588Z 1 Acked 1000 Write 4 Done Sent Done false
118172024-02-06T22:20:38.588Z 2 Acked 1001 Flush 0 Done Sent Done false
118182024-02-06T22:20:38.588Z 3 Acked 1002 Write 10 Done Sent Done false
118192024-02-06T22:20:38.588Z 4 Acked 1003 Write 8 Done Sent Done false
118202024-02-06T22:20:38.588Z 5 Acked 1004 Write 10 Done Sent Done false
118212024-02-06T22:20:38.588Z 6 Acked 1005 Read 10 Done Sent Done false
118222024-02-06T22:20:38.588Z 7 Acked 1006 Write 7 Done Sent Done false
118232024-02-06T22:20:38.589Z 8 Acked 1007 Write 8 Done Sent Done false
118242024-02-06T22:20:38.589Z 9 Acked 1008 Read 8 Done Sent Done false
118252024-02-06T22:20:38.589Z 10 Acked 1009 Read 9 Done Sent Done false
118262024-02-06T22:20:38.589Z 11 Acked 1010 Write 7 Done Sent Done false
118272024-02-06T22:20:38.589Z 12 Acked 1011 Write 6 Done Sent Done false
118282024-02-06T22:20:38.589Z 13 Acked 1012 Read 6 Done Sent Done false
118292024-02-06T22:20:38.589Z 14 Acked 1013 Write 6 Done Sent Done false
118302024-02-06T22:20:38.589Z 15 Acked 1014 Write 4 Done Sent Done false
118312024-02-06T22:20:38.589Z 16 Acked 1015 Write 3 Done Sent Done false
118322024-02-06T22:20:38.589Z 17 Acked 1016 Read 5 Done Sent Done false
118332024-02-06T22:20:38.589Z 18 Acked 1017 Flush 0 Done Sent Done false
118342024-02-06T22:20:38.589Z 19 Acked 1018 Write 3 Done Sent Done false
118352024-02-06T22:20:38.589Z 20 Acked 1019 Read 6 Done Sent Done false
118362024-02-06T22:20:38.589Z 21 Acked 1020 Read 9 Done Sent Done false
118372024-02-06T22:20:38.590Z 22 Acked 1021 Write 6 Done Sent Done false
118382024-02-06T22:20:38.590Z 23 Acked 1022 Read 9 Done Sent Done false
118392024-02-06T22:20:38.590Z 24 Acked 1023 Read 1 Done Sent Done false
118402024-02-06T22:20:38.590Z 25 Acked 1024 Read 7 Done Sent Done false
118412024-02-06T22:20:38.590Z 26 Acked 1025 Read 8 Done Sent Done false
118422024-02-06T22:20:38.590Z 27 Acked 1026 Read 1 Done Sent Done false
118432024-02-06T22:20:38.590Z 28 Acked 1027 Write 9 Done Sent Done false
118442024-02-06T22:20:38.590Z 29 Acked 1028 Write 8 Done Sent Sent false
118452024-02-06T22:20:38.590Z 30 Acked 1029 Write 4 Sent Sent Sent false
118462024-02-06T22:20:38.590Z STATES DS:0 DS:1 DS:2 TOTAL
118472024-02-06T22:20:38.590Z New 0 0 0 0
118482024-02-06T22:20:38.590Z Sent 1 30 2 33
118492024-02-06T22:20:38.590Z Done 29 0 28 57
118502024-02-06T22:20:38.590Z Skipped 0 0 0 0
118512024-02-06T22:20:38.590Z Error 0 0 0 0
118522024-02-06T22:20:38.590ZLast Flush: 1017 0 1017
118532024-02-06T22:20:38.591ZDownstairs last five completed:
118542024-02-06T22:20:38.591ZUpstairs last five completed: 30 29 28 27 26
118552024-02-06T22:20:38.596Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:37.788952331Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1570}
118562024-02-06T22:20:38.596ZWrote out file "/tmp/test_repair_verify.data"
118572024-02-06T22:20:38.596Z
118582024-02-06T22:20:38.599ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
118592024-02-06T22:20:38.623ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118602024-02-06T22:20:38.624Z 0 000-019 108 103 108 204 195 204 F F F <---
118612024-02-06T22:20:38.624Z 1 020-039 102 102 102 193 193 193 F F F
118622024-02-06T22:20:38.624Z 2 040-059 108 105 108 204 200 204 F F F <---
118632024-02-06T22:20:38.624Z 3 060-079 108 106 108 204 201 204 T F T <---
118642024-02-06T22:20:38.624Z 4 080-099 108 106 108 204 201 204 F F F <---
118652024-02-06T22:20:38.624Z 5 100-119 106 106 106 201 201 201 T F T <---
118662024-02-06T22:20:38.624Z 6 120-139 106 106 106 201 201 201 F F F
118672024-02-06T22:20:38.624Z 7 140-159 104 104 104 199 199 199 T F T <---
118682024-02-06T22:20:38.624Z 8 160-179 106 106 106 201 201 201 F F F
118692024-02-06T22:20:38.624Z 9 180-199 106 106 106 201 201 201 T F T <---
118702024-02-06T22:20:38.632Z 10 200-219 106 106 106 201 201 201 F F F
118712024-02-06T22:20:38.632Z 11 220-239 94 94 94 177 177 177 F F F
118722024-02-06T22:20:38.632Z 12 240-259 101 101 101 192 192 192 F F F
118732024-02-06T22:20:38.632Z 13 260-279 107 107 107 202 202 202 F F F
118742024-02-06T22:20:38.632Z 14 280-299 108 100 108 204 189 204 F F F <---
118752024-02-06T22:20:38.632Z 15 300-319 108 103 108 204 195 204 F F F <---
118762024-02-06T22:20:38.632Z 16 320-339 104 104 104 199 199 199 T F T <---
118772024-02-06T22:20:38.633Z 17 340-359 100 100 100 189 189 189 F F F
118782024-02-06T22:20:38.633Z 18 360-379 108 99 108 204 188 204 F F F <---
118792024-02-06T22:20:38.633Z 19 380-399 108 106 108 204 201 204 F F F <---
118802024-02-06T22:20:38.633Z 20 400-419 103 103 103 195 195 195 F F F
118812024-02-06T22:20:38.633Z 21 420-439 108 101 108 204 192 204 F F F <---
118822024-02-06T22:20:38.633Z 22 440-459 108 104 108 204 198 204 F F F <---
118832024-02-06T22:20:38.633Z 23 460-479 108 104 108 203 199 203 F F F <---
118842024-02-06T22:20:38.633Z 24 480-499 108 106 108 204 201 204 F F F <---
118852024-02-06T22:20:38.633Z 25 500-519 104 104 104 199 199 199 F F F
118862024-02-06T22:20:38.633Z 26 520-539 104 104 104 199 199 199 F F F
118872024-02-06T22:20:38.633Z 27 540-559 106 106 106 201 201 201 F F F
118882024-02-06T22:20:38.633Z 28 560-579 104 104 104 196 196 196 F F F
118892024-02-06T22:20:38.633Z 29 580-599 106 106 106 201 201 201 F F F
118902024-02-06T22:20:38.634ZMax gen: 108, Max flush: 204
118912024-02-06T22:20:38.634ZError: Difference in extent metadata found!
118922024-02-06T22:20:38.634ZA Difference in extent metadata is expected here
118932024-02-06T22:20:38.634ZOn loop 54
118942024-02-06T22:20:38.634Z
118952024-02-06T22:20:38.634ZVerifying data now
118962024-02-06T22:20:43.374ZNo change after verify
118972024-02-06T22:20:43.374ZLoop: 54 Downstairs dump after verify (and repair):
118982024-02-06T22:20:43.853ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118992024-02-06T22:20:43.853Z 0 000-019 108 108 108 204 204 204 F F F
119002024-02-06T22:20:43.853Z 1 020-039 102 102 102 193 193 193 F F F
119012024-02-06T22:20:43.853Z 2 040-059 108 108 108 204 204 204 F F F
119022024-02-06T22:20:43.853Z 3 060-079 109 109 109 205 205 205 F F F
119032024-02-06T22:20:43.865Z 4 080-099 108 108 108 204 204 204 F F F
119042024-02-06T22:20:43.865Z 5 100-119 109 109 109 205 205 205 F F F
119052024-02-06T22:20:43.866Z 6 120-139 106 106 106 201 201 201 F F F
119062024-02-06T22:20:43.866Z 7 140-159 109 109 109 205 205 205 F F F
119072024-02-06T22:20:43.866Z 8 160-179 106 106 106 201 201 201 F F F
119082024-02-06T22:20:43.866Z 9 180-199 109 109 109 205 205 205 F F F
119092024-02-06T22:20:43.866Z 10 200-219 106 106 106 201 201 201 F F F
119102024-02-06T22:20:43.866Z 11 220-239 94 94 94 177 177 177 F F F
119112024-02-06T22:20:43.867Z 12 240-259 101 101 101 192 192 192 F F F
119122024-02-06T22:20:43.867Z 13 260-279 107 107 107 202 202 202 F F F
119132024-02-06T22:20:43.867Z 14 280-299 108 108 108 204 204 204 F F F
119142024-02-06T22:20:43.867Z 15 300-319 108 108 108 204 204 204 F F F
119152024-02-06T22:20:43.867Z 16 320-339 109 109 109 205 205 205 F F F
119162024-02-06T22:20:43.868Z 17 340-359 100 100 100 189 189 189 F F F
119172024-02-06T22:20:43.868Z 18 360-379 108 108 108 204 204 204 F F F
119182024-02-06T22:20:43.868Z 19 380-399 108 108 108 204 204 204 F F F
119192024-02-06T22:20:43.868Z 20 400-419 103 103 103 195 195 195 F F F
119202024-02-06T22:20:43.868Z 21 420-439 108 108 108 204 204 204 F F F
119212024-02-06T22:20:43.868Z 22 440-459 108 108 108 204 204 204 F F F
119222024-02-06T22:20:43.868Z 23 460-479 108 108 108 203 203 203 F F F
119232024-02-06T22:20:43.869Z 24 480-499 108 108 108 204 204 204 F F F
119242024-02-06T22:20:43.869Z 25 500-519 104 104 104 199 199 199 F F F
119252024-02-06T22:20:43.869Z 26 520-539 104 104 104 199 199 199 F F F
119262024-02-06T22:20:43.869Z 27 540-559 106 106 106 201 201 201 F F F
119272024-02-06T22:20:43.869Z 28 560-579 104 104 104 196 196 196 F F F
119282024-02-06T22:20:43.869Z 29 580-599 106 106 106 201 201 201 F F F
119292024-02-06T22:20:43.870ZMax gen: 109, Max flush: 205
119302024-02-06T22:20:43.870Z
119312024-02-06T22:20:43.870ZBegin loop 55
119322024-02-06T22:20:43.870ZDownstairs to restart: 0
119332024-02-06T22:20:43.870Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.611538221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578}
119342024-02-06T22:20:43.871Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.611783538Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578}
119352024-02-06T22:20:43.871Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.611805565Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578}
119362024-02-06T22:20:43.871Z{"msg":"Crucible c646e467-5a0f-4a5c-b922-43c9081ecadd has session id: 742ee453-ec03-453a-8095-c6b3ba8ab404","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.611839281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119372024-02-06T22:20:43.871ZCrucible runtime is spawned
119382024-02-06T22:20:43.871Z{"msg":"Upstairs opts: Upstairs UUID: c646e467-5a0f-4a5c-b922-43c9081ecadd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.611897483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119392024-02-06T22:20:43.871Z{"msg":"Crucible stats registered with UUID: c646e467-5a0f-4a5c-b922-43c9081ecadd","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.611942007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119402024-02-06T22:20:43.871Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.611961614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578}
119412024-02-06T22:20:43.872Z{"msg":"c646e467-5a0f-4a5c-b922-43c9081ecadd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612105495Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119422024-02-06T22:20:43.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612156248Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119432024-02-06T22:20:43.872Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612182344Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119442024-02-06T22:20:43.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612218669Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119452024-02-06T22:20:43.872Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612246166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119462024-02-06T22:20:43.872Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612282681Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119472024-02-06T22:20:43.872Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612307417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119482024-02-06T22:20:43.872Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612330094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119492024-02-06T22:20:43.872Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612355471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119502024-02-06T22:20:43.872Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612391736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119512024-02-06T22:20:43.873Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612418342Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119522024-02-06T22:20:43.873Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612443909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119532024-02-06T22:20:43.873Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.612469006Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119542024-02-06T22:20:43.873Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.613156213Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119552024-02-06T22:20:43.873Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.613188338Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119562024-02-06T22:20:43.873Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.613204786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119572024-02-06T22:20:43.873Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.613219494Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119582024-02-06T22:20:43.873Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.61383869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119592024-02-06T22:20:43.873Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.613895473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119602024-02-06T22:20:43.873Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614303738Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119612024-02-06T22:20:43.874Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614384577Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119622024-02-06T22:20:43.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614502921Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119632024-02-06T22:20:43.874Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614541246Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119642024-02-06T22:20:43.874Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614557843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119652024-02-06T22:20:43.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614570872Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119662024-02-06T22:20:43.874Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614598528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119672024-02-06T22:20:43.874Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614621275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119682024-02-06T22:20:43.874Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.614872201Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119692024-02-06T22:20:43.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615336928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119702024-02-06T22:20:43.875Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615372973Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119712024-02-06T22:20:43.875Z{"msg":"[0]R flush_numbers[0..12]: [204, 193, 204, 205, 204, 205, 201, 205, 201, 205, 201, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615390041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119722024-02-06T22:20:43.875Z{"msg":"[0]R generation[0..12]: [108, 102, 108, 109, 108, 109, 106, 109, 106, 109, 106, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615412838Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119732024-02-06T22:20:43.875Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615427986Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119742024-02-06T22:20:43.875Z{"msg":"[1]R flush_numbers[0..12]: [204, 193, 204, 205, 204, 205, 201, 205, 201, 205, 201, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615448983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
119752024-02-06T22:20:43.875Z742ee453-ec03-453a-8095-c6b3ba8ab404"}
119762024-02-06T22:20:43.875Z{"msg":"[1]R generation[0..12]: [108, 102, 108, 109, 108, 109, 106, 109, 106, 109, 106, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.61546691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119772024-02-06T22:20:43.875Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615488758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119782024-02-06T22:20:43.875Z{"msg":"[2]R flush_numbers[0..12]: [204, 193, 204, 205, 204, 205, 201, 205, 201, 205, 201, 177]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615503776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119792024-02-06T22:20:43.875Z{"msg":"[2]R generation[0..12]: [108, 102, 108, 109, 108, 109, 106, 109, 106, 109, 106, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615524653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119802024-02-06T22:20:43.876Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615539621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119812024-02-06T22:20:43.876Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615560098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119822024-02-06T22:20:43.876Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615574396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119832024-02-06T22:20:43.876Z{"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615600972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119842024-02-06T22:20:43.876Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.61562212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119852024-02-06T22:20:43.876Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615648686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119862024-02-06T22:20:43.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615662824Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119872024-02-06T22:20:43.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615684771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119882024-02-06T22:20:43.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615698489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119892024-02-06T22:20:43.876Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615719507Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119902024-02-06T22:20:43.876Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615742903Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119912024-02-06T22:20:43.876Z{"msg":"c646e467-5a0f-4a5c-b922-43c9081ecadd is now active with session: 742ee453-ec03-453a-8095-c6b3ba8ab404","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.6157664Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119922024-02-06T22:20:43.876Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615793317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578,"session_id":"742ee453-ec03-453a-8095-c6b3ba8ab404"}
119932024-02-06T22:20:43.877Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:42.615815834Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578}
119942024-02-06T22:20:43.877ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
119952024-02-06T22:20:43.877ZLoading write count information from file "/tmp/test_repair_verify.data"
119962024-02-06T22:20:43.877ZRun Repair workload
119972024-02-06T22:20:43.877Z01/30 Write block 249 len 4096 data: 13
119982024-02-06T22:20:43.877Z02/30 Flush
119992024-02-06T22:20:43.877Z03/30 Write block 94 len 40960 data: 6 7 7 7 8 8 8 5 6 5
120002024-02-06T22:20:43.877Z04/30 Read block 206 len 12288
120012024-02-06T22:20:43.877Z05/30 Write block 589 len 8192 data: 15 18
120022024-02-06T22:20:43.877Z06/30 Write block 219 len 36864 data: 9 14 14 13 12 12 9 9 7
120032024-02-06T22:20:43.887Z07/30 Read block 568 len 8192
120042024-02-06T22:20:43.887Z08/30 Flush
120052024-02-06T22:20:43.887Z09/30 Flush
120062024-02-06T22:20:43.887Z10/30 Write block 159 len 16384 data: 9 8 7 8
120072024-02-06T22:20:43.888Z11/30 Write block 444 len 20480 data: 6 6 5 5 7
120082024-02-06T22:20:43.888Z12/30 Write block 556 len 32768 data: 10 10 11 8 7 7 8 7
120092024-02-06T22:20:43.888Z13/30 Write block 551 len 12288 data: 7 8 8
120102024-02-06T22:20:43.888Z14/30 Read block 492 len 16384
120112024-02-06T22:20:43.888Z15/30 Write block 182 len 40960 data: 12 13 15 15 16 14 10 9 7 8
120122024-02-06T22:20:43.889Z16/30 Flush
120132024-02-06T22:20:43.889Z17/30 Write block 215 len 8192 data: 11 9
120142024-02-06T22:20:43.889Z18/30 Read block 394 len 4096
120152024-02-06T22:20:43.889Z19/30 Read block 223 len 40960
120162024-02-06T22:20:43.890Z20/30 Read block 575 len 36864
120172024-02-06T22:20:43.890Z21/30 Read block 338 len 28672
120182024-02-06T22:20:43.890Z22/30 Flush
120192024-02-06T22:20:43.890Z23/30 Write block 127 len 32768 data: 13 14 13 14 13 10 10 9
120202024-02-06T22:20:43.891Z24/30 Write block 331 len 28672 data: 6 6 9 12 11 12 11
120212024-02-06T22:20:43.891Z25/30 Read block 235 len 20480
120222024-02-06T22:20:43.891Z26/30 Write block 270 len 36864 data: 11 9 9 6 6 8 10 10 11
120232024-02-06T22:20:43.891Z27/30 Write block 155 len 32768 data: 13 11 13 9 10 9 8 9
120242024-02-06T22:20:43.891Z28/30 Write block 554 len 4096 data: 7
120252024-02-06T22:20:43.892Z29/30 Read block 237 len 36864
120262024-02-06T22:20:43.892Z30/30 Write block 425 len 20480 data: 9 10 8 9 8
120272024-02-06T22:20:43.892Z----------------------------------------------------------------
120282024-02-06T22:20:43.892Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:28
120292024-02-06T22:20:43.893ZStates: Active Active Active
120302024-02-06T22:20:43.893ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
120312024-02-06T22:20:43.893Z 3 Acked 1002 Write 10 Done Done Done false
120322024-02-06T22:20:43.893Z 4 Acked 1003 Read 3 Done Done Done false
120332024-02-06T22:20:43.893Z 5 Acked 1004 Write 2 Sent Done Done false
120342024-02-06T22:20:43.893Z 6 Acked 1005 Write 9 Sent Done Done false
120352024-02-06T22:20:43.894Z 7 Acked 1006 Read 2 Sent Done Done false
120362024-02-06T22:20:43.894Z 8 Acked 1007 Flush 0 Sent Done Done false
120372024-02-06T22:20:43.894Z 9 Acked 1008 Flush 0 Sent Done Done false
120382024-02-06T22:20:43.894Z 10 Acked 1009 Write 4 Sent Done Done false
120392024-02-06T22:20:43.894Z 11 Acked 1010 Write 5 Sent Done Done false
120402024-02-06T22:20:43.895Z 12 Acked 1011 Write 8 Sent Done Done false
120412024-02-06T22:20:43.895Z 13 Acked 1012 Write 3 Sent Done Done false
120422024-02-06T22:20:43.895Z 14 Acked 1013 Read 4 Sent Done Done false
120432024-02-06T22:20:43.895Z 15 Acked 1014 Write 10 Sent Done Done false
120442024-02-06T22:20:43.895Z 16 Acked 1015 Flush 0 Sent Done Done false
120452024-02-06T22:20:43.896Z 17 Acked 1016 Write 2 Sent Done Done false
120462024-02-06T22:20:43.896Z 18 Acked 1017 Read 1 Sent Done Done false
120472024-02-06T22:20:43.896Z 19 Acked 1018 Read 10 Sent Done Done false
120482024-02-06T22:20:43.896Z 20 Acked 1019 Read 9 Sent Done Done false
120492024-02-06T22:20:43.896Z 21 Acked 1020 Read 7 Sent Done Done false
120502024-02-06T22:20:43.896Z 22 Acked 1021 Flush 0 Sent Done Done false
120512024-02-06T22:20:43.897Z 23 Acked 1022 Write 8 Sent Done Done false
120522024-02-06T22:20:43.897Z 24 Acked 1023 Write 7 Sent Done Done false
120532024-02-06T22:20:43.897Z 25 Acked 1024 Read 5 Sent Done Done false
120542024-02-06T22:20:43.897Z 26 Acked 1025 Write 9 Sent Done Done false
120552024-02-06T22:20:43.897Z 27 Acked 1026 Write 8 Sent Done Done false
120562024-02-06T22:20:43.897Z 28 Acked 1027 Write 1 Sent Done Done false
120572024-02-06T22:20:43.897Z 29 Acked 1028 Read 9 Sent Done Done false
120582024-02-06T22:20:43.898Z 30 Acked 1029 Write 5 Sent Sent Sent false
120592024-02-06T22:20:43.898Z STATES DS:0 DS:1 DS:2 TOTAL
120602024-02-06T22:20:43.898Z New 0 0 0 0
120612024-02-06T22:20:43.898Z Sent 26 1 1 28
120622024-02-06T22:20:43.898Z Done 2 27 27 56
120632024-02-06T22:20:43.898Z Skipped 0 0 0 0
120642024-02-06T22:20:43.898Z Error 0 0 0 0
120652024-02-06T22:20:43.898ZLast Flush: 1001 1021 1021
120662024-02-06T22:20:43.898ZDownstairs last five completed: 1001 1000
120672024-02-06T22:20:43.899ZUpstairs last five completed: 30 29 28 27 26
120682024-02-06T22:20:43.899Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:42.647300969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1578}
120692024-02-06T22:20:43.899ZWrote out file "/tmp/test_repair_verify.data"
120702024-02-06T22:20:43.899Z
120712024-02-06T22:20:43.899ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
120722024-02-06T22:20:43.899ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120732024-02-06T22:20:43.899Z 0 000-019 108 108 108 204 204 204 F F F
120742024-02-06T22:20:43.899Z 1 020-039 102 102 102 193 193 193 F F F
120752024-02-06T22:20:43.899Z 2 040-059 108 108 108 204 204 204 F F F
120762024-02-06T22:20:43.899Z 3 060-079 109 109 109 205 205 205 F F F
120772024-02-06T22:20:43.899Z 4 080-099 108 110 110 204 207 207 T F F <---
120782024-02-06T22:20:43.900Z 5 100-119 109 110 110 205 207 207 T F F <---
120792024-02-06T22:20:43.900Z 6 120-139 106 106 106 201 201 201 F T T <---
120802024-02-06T22:20:43.900Z 7 140-159 109 110 110 205 209 209 F T T <---
120812024-02-06T22:20:43.900Z 8 160-179 106 110 110 201 209 209 F T T <---
120822024-02-06T22:20:43.900Z 9 180-199 109 110 110 205 209 209 F F F <---
120832024-02-06T22:20:43.901Z 10 200-219 106 110 110 201 210 210 F F F <---
120842024-02-06T22:20:43.901Z 11 220-239 94 110 110 177 207 207 F F F <---
120852024-02-06T22:20:43.902Z 12 240-259 110 110 110 206 206 206 F F F
120862024-02-06T22:20:43.902Z 13 260-279 107 107 107 202 202 202 F T T <---
120872024-02-06T22:20:43.902Z 14 280-299 108 108 108 204 204 204 F F F
120882024-02-06T22:20:43.902Z 15 300-319 108 108 108 204 204 204 F F F
120892024-02-06T22:20:43.902Z 16 320-339 109 109 109 205 205 205 F T T <---
120902024-02-06T22:20:43.903Z 17 340-359 100 100 100 189 189 189 F F F
120912024-02-06T22:20:43.903Z 18 360-379 108 108 108 204 204 204 F F F
120922024-02-06T22:20:43.903Z 19 380-399 108 108 108 204 204 204 F F F
120932024-02-06T22:20:43.903Z 20 400-419 103 103 103 195 195 195 F F F
120942024-02-06T22:20:43.903Z 21 420-439 108 108 108 204 204 204 F T T <---
120952024-02-06T22:20:43.904Z 22 440-459 108 110 110 204 209 209 F F F <---
120962024-02-06T22:20:43.904Z 23 460-479 108 108 108 203 203 203 F F F
120972024-02-06T22:20:43.904Z 24 480-499 108 108 108 204 204 204 F F F
120982024-02-06T22:20:43.904Z 25 500-519 104 104 104 199 199 199 F F F
120992024-02-06T22:20:43.904Z 26 520-539 104 104 104 199 199 199 F F F
121002024-02-06T22:20:43.905Z 27 540-559 106 110 110 201 209 209 F T T <---
121012024-02-06T22:20:43.905Z 28 560-579 104 110 110 196 209 209 F F F <---
121022024-02-06T22:20:43.905Z 29 580-599 106 110 110 201 207 207 F F F <---
121032024-02-06T22:20:43.914ZMax gen: 110, Max flush: 210
121042024-02-06T22:20:43.914ZError: Difference in extent metadata found!
121052024-02-06T22:20:43.914ZA Difference in extent metadata is expected here
121062024-02-06T22:20:43.914ZOn loop 55
121072024-02-06T22:20:43.915Z
121082024-02-06T22:20:43.915ZVerifying data now
121092024-02-06T22:20:49.209ZNo change after verify
121102024-02-06T22:20:49.209ZLoop: 55 Downstairs dump after verify (and repair):
121112024-02-06T22:20:49.227ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121122024-02-06T22:20:49.227Z 0 000-019 108 108 108 204 204 204 F F F
121132024-02-06T22:20:49.227Z 1 020-039 102 102 102 193 193 193 F F F
121142024-02-06T22:20:49.227Z 2 040-059 108 108 108 204 204 204 F F F
121152024-02-06T22:20:49.228Z 3 060-079 109 109 109 205 205 205 F F F
121162024-02-06T22:20:49.228Z 4 080-099 110 110 110 207 207 207 F F F
121172024-02-06T22:20:49.228Z 5 100-119 110 110 110 207 207 207 F F F
121182024-02-06T22:20:49.228Z 6 120-139 111 111 111 211 211 211 F F F
121192024-02-06T22:20:49.235Z 7 140-159 111 111 111 211 211 211 F F F
121202024-02-06T22:20:49.235Z 8 160-179 111 111 111 211 211 211 F F F
121212024-02-06T22:20:49.235Z 9 180-199 110 110 110 209 209 209 F F F
121222024-02-06T22:20:49.236Z 10 200-219 110 110 110 210 210 210 F F F
121232024-02-06T22:20:49.236Z 11 220-239 110 110 110 207 207 207 F F F
121242024-02-06T22:20:49.236Z 12 240-259 110 110 110 206 206 206 F F F
121252024-02-06T22:20:49.236Z 13 260-279 111 111 111 211 211 211 F F F
121262024-02-06T22:20:49.236Z 14 280-299 108 108 108 204 204 204 F F F
121272024-02-06T22:20:49.236Z 15 300-319 108 108 108 204 204 204 F F F
121282024-02-06T22:20:49.236Z 16 320-339 111 111 111 211 211 211 F F F
121292024-02-06T22:20:49.236Z 17 340-359 100 100 100 189 189 189 F F F
121302024-02-06T22:20:49.237Z 18 360-379 108 108 108 204 204 204 F F F
121312024-02-06T22:20:49.237Z 19 380-399 108 108 108 204 204 204 F F F
121322024-02-06T22:20:49.237Z 20 400-419 103 103 103 195 195 195 F F F
121332024-02-06T22:20:49.237Z 21 420-439 111 111 111 211 211 211 F F F
121342024-02-06T22:20:49.237Z 22 440-459 110 110 110 209 209 209 F F F
121352024-02-06T22:20:49.237Z 23 460-479 108 108 108 203 203 203 F F F
121362024-02-06T22:20:49.237Z 24 480-499 108 108 108 204 204 204 F F F
121372024-02-06T22:20:49.237Z 25 500-519 104 104 104 199 199 199 F F F
121382024-02-06T22:20:49.237Z 26 520-539 104 104 104 199 199 199 F F F
121392024-02-06T22:20:49.237Z 27 540-559 111 111 111 211 211 211 F F F
121402024-02-06T22:20:49.237Z 28 560-579 110 110 110 209 209 209 F F F
121412024-02-06T22:20:49.238Z 29 580-599 110 110 110 207 207 207 F F F
121422024-02-06T22:20:49.238ZMax gen: 111, Max flush: 211
121432024-02-06T22:20:49.238Z
121442024-02-06T22:20:49.238ZBegin loop 56
121452024-02-06T22:20:49.238ZDownstairs to restart: 1
121462024-02-06T22:20:49.251Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.444371956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586}
121472024-02-06T22:20:49.251Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.444651308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586Crucible runtime is spawned
121482024-02-06T22:20:49.252Z}
121492024-02-06T22:20:49.252Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.444690593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586}
121502024-02-06T22:20:49.252Z{"msg":"Crucible da9dcf46-aae8-4901-8725-a5af8a54d0e8 has session id: 190d3da1-1101-40db-95cb-d54ef030c9ae","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.444714249Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121512024-02-06T22:20:49.265Z{"msg":"Upstairs opts: Upstairs UUID: da9dcf46-aae8-4901-8725-a5af8a54d0e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.444753914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121522024-02-06T22:20:49.265Z{"msg":"Crucible stats registered with UUID: da9dcf46-aae8-4901-8725-a5af8a54d0e8","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44478464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121532024-02-06T22:20:49.265Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.444827564Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586}
121542024-02-06T22:20:49.265Z{"msg":"da9dcf46-aae8-4901-8725-a5af8a54d0e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44493525Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121552024-02-06T22:20:49.265Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.444984373Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121562024-02-06T22:20:49.265Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445014529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121572024-02-06T22:20:49.265Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445060643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121582024-02-06T22:20:49.265Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445088769Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121592024-02-06T22:20:49.265Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445127114Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121602024-02-06T22:20:49.265Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44515455Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121612024-02-06T22:20:49.265Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445194375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121622024-02-06T22:20:49.265Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44522799Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121632024-02-06T22:20:49.265Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445346664Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121642024-02-06T22:20:49.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445370041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121652024-02-06T22:20:49.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445397517Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121662024-02-06T22:20:49.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.445418834Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121672024-02-06T22:20:49.265Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.446095203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121682024-02-06T22:20:49.265Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.446134597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121692024-02-06T22:20:49.266Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.446151985Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121702024-02-06T22:20:49.266Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44618797Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121712024-02-06T22:20:49.266Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.446776381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121722024-02-06T22:20:49.266Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.446819655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121732024-02-06T22:20:49.266Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447146381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121742024-02-06T22:20:49.266Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447188405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121752024-02-06T22:20:49.266Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447339025Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121762024-02-06T22:20:49.266Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44737583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121772024-02-06T22:20:49.266Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447401057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121782024-02-06T22:20:49.266Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447433352Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121792024-02-06T22:20:49.266Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447458889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121802024-02-06T22:20:49.266Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447486665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121812024-02-06T22:20:49.266Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.447654062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121822024-02-06T22:20:49.266Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448067196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121832024-02-06T22:20:49.266Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448107941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121842024-02-06T22:20:49.266Z{"msg":"[0]R flush_numbers[0..12]: [204, 193, 204, 205, 207, 207, 211, 211, 211, 209, 210, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448124679Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121852024-02-06T22:20:49.266Z{"msg":"[0]R generation[0..12]: [108, 102, 108, 109, 110, 110, 111, 111, 111, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448158944Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121862024-02-06T22:20:49.266Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44819111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121872024-02-06T22:20:49.266Z{"msg":"[1]R flush_numbers[0..12]: [204, 193, 204, 205, 207, 207, 211, 211, 211, 209, 210, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448225695Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121882024-02-06T22:20:49.266Z{"msg":"[1]R generation[0..12]: [108, 102, 108, 109, 110, 110, 111, 111, 111, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448245492Z","Wait for a query_work_queue command to finish before sending IO
121892024-02-06T22:20:49.266Zhostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121902024-02-06T22:20:49.266Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448275318Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121912024-02-06T22:20:49.266Z{"msg":"[2]R flush_numbers[0..12]: [204, 193, 204, 205, 207, 207, 211, 211, 211, 209, 210, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448295486Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121922024-02-06T22:20:49.266Z{"msg":"[2]R generation[0..12]: [108, 102, 108, 109, 110, 110, 111, 111, 111, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448329531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121932024-02-06T22:20:49.267Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448344949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121942024-02-06T22:20:49.267Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448381714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121952024-02-06T22:20:49.267Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448399502Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121962024-02-06T22:20:49.267Z{"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448429487Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121972024-02-06T22:20:49.267Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448445305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121982024-02-06T22:20:49.267Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448476661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
121992024-02-06T22:20:49.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448492109Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
122002024-02-06T22:20:49.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448523195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
122012024-02-06T22:20:49.267Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448539803Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
122022024-02-06T22:20:49.267Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448570168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
122032024-02-06T22:20:49.267Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448586586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
122042024-02-06T22:20:49.267Z{"msg":"da9dcf46-aae8-4901-8725-a5af8a54d0e8 is now active with session: 190d3da1-1101-40db-95cb-d54ef030c9ae","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448615542Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
122052024-02-06T22:20:49.267Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.44863008Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586,"session_id":"190d3da1-1101-40db-95cb-d54ef030c9ae"}
122062024-02-06T22:20:49.267Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:48.448660336Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586}
122072024-02-06T22:20:49.267ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122082024-02-06T22:20:49.267ZLoading write count information from file "/tmp/test_repair_verify.data"
122092024-02-06T22:20:49.267ZRun Repair workload
122102024-02-06T22:20:49.267Z01/30 Write block 510 len 24576 data: 9 8 7 11 11 10
122112024-02-06T22:20:49.267Z02/30 Read block 542 len 8192
122122024-02-06T22:20:49.267Z03/30 Read block 377 len 36864
122132024-02-06T22:20:49.267Z04/30 Write block 178 len 28672 data: 11 15 14 14 13 14 16
122142024-02-06T22:20:49.267Z05/30 Flush
122152024-02-06T22:20:49.267Z06/30 Write block 309 len 24576 data: 8 10 11 11 12 12
122162024-02-06T22:20:49.267Z07/30 Read block 176 len 32768
122172024-02-06T22:20:49.268Z08/30 Read block 244 len 28672
122182024-02-06T22:20:49.271Z09/30 Write block 578 len 16384 data: 12 11 10 7
122192024-02-06T22:20:49.271Z10/30 Write block 456 len 36864 data: 6 7 7 7 9 7 7 6 6
122202024-02-06T22:20:49.271Z11/30 Write block 484 len 4096 data: 10
122212024-02-06T22:20:49.271Z12/30 Read block 232 len 36864
122222024-02-06T22:20:49.272Z13/30 Write block 317 len 32768 data: 10 9 9 7 7 8 6 6
122232024-02-06T22:20:49.272Z14/30 Read block 285 len 4096
122242024-02-06T22:20:49.272Z15/30 Write block 133 len 24576 data: 11 10 8 7 7 8
122252024-02-06T22:20:49.272Z16/30 Read block 189 len 24576
122262024-02-06T22:20:49.272Z17/30 Read block 28 len 16384
122272024-02-06T22:20:49.272Z18/30 Write block 460 len 20480 data: 10 8 8 7 7
122282024-02-06T22:20:49.272Z19/30 Write block 489 len 40960 data: 7 5 6 8 9 9 9 9 9 10
122292024-02-06T22:20:49.272Z20/30 Write block 409 len 28672 data: 11 11 12 9 8 8 11
122302024-02-06T22:20:49.272Z21/30 Read block 570 len 12288
122312024-02-06T22:20:49.272Z22/30 Write block 342 len 4096 data: 9
122322024-02-06T22:20:49.272Z23/30 Write block 352 len 12288 data: 17 15 14
122332024-02-06T22:20:49.272Z24/30 Read block 272 len 36864
122342024-02-06T22:20:49.275Z25/30 Flush
122352024-02-06T22:20:49.278Z26/30 Write block 489 len 16384 data: 8 6 7 9
122362024-02-06T22:20:49.278Z27/30 Read block 429 len 20480
122372024-02-06T22:20:49.283Z28/30 Read block 58 len 24576
122382024-02-06T22:20:49.283Z29/30 Read block 431 len 24576
122392024-02-06T22:20:49.283Z30/30 Read block 414 len 4096
122402024-02-06T22:20:49.283Z----------------------------------------------------------------
122412024-02-06T22:20:49.283Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
122422024-02-06T22:20:49.283ZStates: Active Active Active
122432024-02-06T22:20:49.283ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122442024-02-06T22:20:49.283Z 1 Acked 1000 Write 6 Done Done Done false
122452024-02-06T22:20:49.283Z 2 Acked 1001 Read 2 Done Done Done false
122462024-02-06T22:20:49.283Z 3 Acked 1002 Read 9 Done Sent Done false
122472024-02-06T22:20:49.283Z 4 Acked 1003 Write 7 Done Sent Done false
122482024-02-06T22:20:49.283Z 5 Acked 1004 Flush 0 Done Sent Done false
122492024-02-06T22:20:49.283Z 6 Acked 1005 Write 6 Done Sent Done false
122502024-02-06T22:20:49.283Z 7 Acked 1006 Read 8 Done Sent Done false
122512024-02-06T22:20:49.283Z 8 Acked 1007 Read 7 Done Sent Done false
122522024-02-06T22:20:49.283Z 9 Acked 1008 Write 4 Done Sent Done false
122532024-02-06T22:20:49.283Z 10 Acked 1009 Write 9 Done Sent Done false
122542024-02-06T22:20:49.283Z 11 Acked 1010 Write 1 Done Sent Done false
122552024-02-06T22:20:49.283Z 12 Acked 1011 Read 9 Done Sent Done false
122562024-02-06T22:20:49.283Z 13 Acked 1012 Write 8 Done Sent Done false
122572024-02-06T22:20:49.284Z 14 Acked 1013 Read 1 Done Sent Done false
122582024-02-06T22:20:49.284Z 15 Acked 1014 Write 6 Done Sent Done false
122592024-02-06T22:20:49.284Z 16 Acked 1015 Read 6 Done Sent Done false
122602024-02-06T22:20:49.284Z 17 Acked 1016 Read 4 Done Sent Done false
122612024-02-06T22:20:49.284Z 18 Acked 1017 Write 5 Done Sent Done false
122622024-02-06T22:20:49.284Z 19 Acked 1018 Write 10 Done Sent Done false
122632024-02-06T22:20:49.284Z 20 Acked 1019 Write 7 Done Sent Done false
122642024-02-06T22:20:49.284Z 21 Acked 1020 Read 3 Done Sent Done false
122652024-02-06T22:20:49.284Z 22 Acked 1021 Write 1 Done Sent Done false
122662024-02-06T22:20:49.284Z 23 Acked 1022 Write 3 Done Sent Done false
122672024-02-06T22:20:49.284Z 24 Acked 1023 Read 9 Done Sent Done false
122682024-02-06T22:20:49.284Z 25 Acked 1024 Flush 0 Done Sent Done false
122692024-02-06T22:20:49.284Z 26 Acked 1025 Write 4 Done Sent Done false
122702024-02-06T22:20:49.284Z 27 Acked 1026 Read 5 Done Sent Done false
122712024-02-06T22:20:49.284Z 28 Acked 1027 Read 6 Done Sent Done false
122722024-02-06T22:20:49.284Z 29 Acked 1028 Read 6 Done Sent Done false
122732024-02-06T22:20:49.284Z 30 Acked 1029 Read 1 Done Sent Done false
122742024-02-06T22:20:49.284Z STATES DS:0 DS:1 DS:2 TOTAL
122752024-02-06T22:20:49.284Z New 0 0 0 0
122762024-02-06T22:20:49.284Z Sent 0 28 0 28
122772024-02-06T22:20:49.284Z Done 30 2 30 62
122782024-02-06T22:20:49.284Z Skipped 0 0 0 0
122792024-02-06T22:20:49.284Z Error 0 0 0 0
122802024-02-06T22:20:49.284ZLast Flush: 1024 0 1024
122812024-02-06T22:20:49.284ZDownstairs last five completed:
122822024-02-06T22:20:49.285ZUpstairs last five completed: 30 29 28 27 26
122832024-02-06T22:20:49.285Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:48.481689804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1586}
122842024-02-06T22:20:49.285ZWrote out file "/tmp/test_repair_verify.data"
122852024-02-06T22:20:49.288Z
122862024-02-06T22:20:49.291ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
122872024-02-06T22:20:49.303ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122882024-02-06T22:20:49.303Z 0 000-019 108 108 108 204 204 204 F F F
122892024-02-06T22:20:49.303Z 1 020-039 102 102 102 193 193 193 F F F
122902024-02-06T22:20:49.304Z 2 040-059 108 108 108 204 204 204 F F F
122912024-02-06T22:20:49.304Z 3 060-079 109 109 109 205 205 205 F F F
122922024-02-06T22:20:49.304Z 4 080-099 110 110 110 207 207 207 F F F
122932024-02-06T22:20:49.304Z 5 100-119 110 110 110 207 207 207 F F F
122942024-02-06T22:20:49.307Z 6 120-139 112 111 112 213 211 213 F F F <---
122952024-02-06T22:20:49.307Z 7 140-159 111 111 111 211 211 211 F F F
122962024-02-06T22:20:49.307Z 8 160-179 112 111 112 212 211 212 F F F <---
122972024-02-06T22:20:49.307Z 9 180-199 112 110 112 212 209 212 F F F <---
122982024-02-06T22:20:49.307Z 10 200-219 110 110 110 210 210 210 F F F
122992024-02-06T22:20:49.307Z 11 220-239 110 110 110 207 207 207 F F F
123002024-02-06T22:20:49.307Z 12 240-259 110 110 110 206 206 206 F F F
123012024-02-06T22:20:49.307Z 13 260-279 111 111 111 211 211 211 F F F
123022024-02-06T22:20:49.307Z 14 280-299 108 108 108 204 204 204 F F F
123032024-02-06T22:20:49.307Z 15 300-319 112 108 112 213 204 213 F F F <---
123042024-02-06T22:20:49.307Z 16 320-339 112 111 112 213 211 213 F F F <---
123052024-02-06T22:20:49.307Z 17 340-359 112 100 112 213 189 213 F F F <---
123062024-02-06T22:20:49.308Z 18 360-379 108 108 108 204 204 204 F F F
123072024-02-06T22:20:49.308Z 19 380-399 108 108 108 204 204 204 F F F
123082024-02-06T22:20:49.308Z 20 400-419 112 103 112 213 195 213 F F F <---
123092024-02-06T22:20:49.308Z 21 420-439 111 111 111 211 211 211 F F F
123102024-02-06T22:20:49.308Z 22 440-459 112 110 112 213 209 213 F F F <---
123112024-02-06T22:20:49.308Z 23 460-479 112 108 112 213 203 213 F F F <---
123122024-02-06T22:20:49.308Z 24 480-499 112 108 112 213 204 213 T F T <---
123132024-02-06T22:20:49.308Z 25 500-519 112 104 112 212 199 212 F T F <---
123142024-02-06T22:20:49.308Z 26 520-539 104 104 104 199 199 199 F F F
123152024-02-06T22:20:49.308Z 27 540-559 111 111 111 211 211 211 F F F
123162024-02-06T22:20:49.308Z 28 560-579 112 110 112 213 209 213 F F F <---
123172024-02-06T22:20:49.308Z 29 580-599 112 110 112 213 207 213 F F F <---
123182024-02-06T22:20:49.308ZMax gen: 112, Max flush: 213
123192024-02-06T22:20:49.308ZError: Difference in extent metadata found!
123202024-02-06T22:20:49.308ZA Difference in extent metadata is expected here
123212024-02-06T22:20:49.308ZOn loop 56
123222024-02-06T22:20:49.308Z
123232024-02-06T22:20:49.312ZVerifying data now
123242024-02-06T22:20:54.943ZNo change after verify
123252024-02-06T22:20:54.943ZLoop: 56 Downstairs dump after verify (and repair):
123262024-02-06T22:20:54.959ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123272024-02-06T22:20:54.959Z 0 000-019 108 108 108 204 204 204 F F F
123282024-02-06T22:20:54.960Z 1 020-039 102 102 102 193 193 193 F F F
123292024-02-06T22:20:54.960Z 2 040-059 108 108 108 204 204 204 F F F
123302024-02-06T22:20:54.960Z 3 060-079 109 109 109 205 205 205 F F F
123312024-02-06T22:20:54.960Z 4 080-099 110 110 110 207 207 207 F F F
123322024-02-06T22:20:54.960Z 5 100-119 110 110 110 207 207 207 F F F
123332024-02-06T22:20:54.960Z 6 120-139 112 112 112 213 213 213 F F F
123342024-02-06T22:20:54.960Z 7 140-159 111 111 111 211 211 211 F F F
123352024-02-06T22:20:54.960Z 8 160-179 112 112 112 212 212 212 F F F
123362024-02-06T22:20:54.960Z 9 180-199 112 112 112 212 212 212 F F F
123372024-02-06T22:20:54.961Z 10 200-219 110 110 110 210 210 210 F F F
123382024-02-06T22:20:54.961Z 11 220-239 110 110 110 207 207 207 F F F
123392024-02-06T22:20:54.961Z 12 240-259 110 110 110 206 206 206 F F F
123402024-02-06T22:20:54.961Z 13 260-279 111 111 111 211 211 211 F F F
123412024-02-06T22:20:54.978Z 14 280-299 108 108 108 204 204 204 F F F
123422024-02-06T22:20:54.978Z 15 300-319 112 112 112 213 213 213 F F F
123432024-02-06T22:20:54.978Z 16 320-339 112 112 112 213 213 213 F F F
123442024-02-06T22:20:54.979Z 17 340-359 112 112 112 213 213 213 F F F
123452024-02-06T22:20:54.979Z 18 360-379 108 108 108 204 204 204 F F F
123462024-02-06T22:20:54.979Z 19 380-399 108 108 108 204 204 204 F F F
123472024-02-06T22:20:54.979Z 20 400-419 112 112 112 213 213 213 F F F
123482024-02-06T22:20:54.979Z 21 420-439 111 111 111 211 211 211 F F F
123492024-02-06T22:20:54.979Z 22 440-459 112 112 112 213 213 213 F F F
123502024-02-06T22:20:54.979Z 23 460-479 112 112 112 213 213 213 F F F
123512024-02-06T22:20:54.979Z 24 480-499 113 113 113 214 214 214 F F F
123522024-02-06T22:20:54.979Z 25 500-519 112 112 112 212 212 212 F F F
123532024-02-06T22:20:54.980Z 26 520-539 104 104 104 199 199 199 F F F
123542024-02-06T22:20:54.980Z 27 540-559 111 111 111 211 211 211 F F F
123552024-02-06T22:20:54.980Z 28 560-579 112 112 112 213 213 213 F F F
123562024-02-06T22:20:54.980Z 29 580-599 112 112 112 213 213 213 F F F
123572024-02-06T22:20:54.980ZMax gen: 113, Max flush: 214
123582024-02-06T22:20:54.980Z
123592024-02-06T22:20:54.980ZBegin loop 57
123602024-02-06T22:20:54.980ZDownstairs to restart: 2
123612024-02-06T22:20:55.427Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169287368Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594}
123622024-02-06T22:20:55.428Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169536214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594}
123632024-02-06T22:20:55.428Z{"msg":"Crucible runtime is spawned
123642024-02-06T22:20:55.428ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169614294Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594}
123652024-02-06T22:20:55.428Z{"msg":"Crucible 94db21f0-f738-482b-8fc7-7009bb3ef1dd has session id: 18bc9466-4d6d-49cc-a978-37aa010ce4c3","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.16964162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123662024-02-06T22:20:55.428Z{"msg":"Upstairs opts: Upstairs UUID: 94db21f0-f738-482b-8fc7-7009bb3ef1dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169691493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123672024-02-06T22:20:55.428Z{"msg":"Crucible stats registered with UUID: 94db21f0-f738-482b-8fc7-7009bb3ef1dd","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169725429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123682024-02-06T22:20:55.429Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169748166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594}
123692024-02-06T22:20:55.429Z{"msg":"94db21f0-f738-482b-8fc7-7009bb3ef1dd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169837153Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123702024-02-06T22:20:55.429Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169880568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123712024-02-06T22:20:55.429Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169903665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123722024-02-06T22:20:55.429Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169934551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123732024-02-06T22:20:55.429Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.169969146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123742024-02-06T22:20:55.429Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170003401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123752024-02-06T22:20:55.429Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170050745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123762024-02-06T22:20:55.430Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170097309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123772024-02-06T22:20:55.430Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170148262Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123782024-02-06T22:20:55.430Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170186117Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123792024-02-06T22:20:55.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170215593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123802024-02-06T22:20:55.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170253957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123812024-02-06T22:20:55.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170294822Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123822024-02-06T22:20:55.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170915718Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123832024-02-06T22:20:55.431Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170955933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123842024-02-06T22:20:55.431Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.170981579Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123852024-02-06T22:20:55.431Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.171006866Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123862024-02-06T22:20:55.431Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.171597176Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123872024-02-06T22:20:55.431Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.171656018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123882024-02-06T22:20:55.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172277754Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123892024-02-06T22:20:55.432Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172326237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123902024-02-06T22:20:55.432Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172351074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123912024-02-06T22:20:55.432Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.17238206Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123922024-02-06T22:20:55.432Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172415245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123932024-02-06T22:20:55.432Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172438702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123942024-02-06T22:20:55.432Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172469528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123952024-02-06T22:20:55.432Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172486426Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123962024-02-06T22:20:55.433Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.172918547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123972024-02-06T22:20:55.433Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173370266Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123982024-02-06T22:20:55.433Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173409481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
123992024-02-06T22:20:55.433Z{"msg":"[0]R flush_numbers[0..12]: [204, 193, 204, 205, 207, 207, 213, 211, 212, 212, 210, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173427468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124002024-02-06T22:20:55.433Z{"msg":"[0]R generation[0..12]: [108, 102, 108, 109, 110, 110, 112, 111, 112, 112, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173450675Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124012024-02-06T22:20:55.433Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173466143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124022024-02-06T22:20:55.434Z{"msg":"[1]R flush_numbers[0..12]: [204, 193, 204, 205, 207, 207, 213, 211, 212, 212, 210, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.17348823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
124032024-02-06T22:20:55.434Z18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124042024-02-06T22:20:55.434Z{"msg":"[1]R generation[0..12]: [108, 102, 108, 109, 110, 110, 112, 111, 112, 112, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173506038Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124052024-02-06T22:20:55.434Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173528885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124062024-02-06T22:20:55.434Z{"msg":"[2]R flush_numbers[0..12]: [204, 193, 204, 205, 207, 207, 213, 211, 212, 212, 210, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173544583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124072024-02-06T22:20:55.434Z{"msg":"[2]R generation[0..12]: [108, 102, 108, 109, 110, 110, 112, 111, 112, 112, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.17356603Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124082024-02-06T22:20:55.434Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173581788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124092024-02-06T22:20:55.435Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173601005Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124102024-02-06T22:20:55.435Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173620882Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124112024-02-06T22:20:55.435Z{"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173651918Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124122024-02-06T22:20:55.435Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173669826Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124132024-02-06T22:20:55.435Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173700682Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124142024-02-06T22:20:55.436Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173716389Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124152024-02-06T22:20:55.436Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173747755Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124162024-02-06T22:20:55.436Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173767063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124172024-02-06T22:20:55.436Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173796689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124182024-02-06T22:20:55.436Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173812176Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124192024-02-06T22:20:55.436Z{"msg":"94db21f0-f738-482b-8fc7-7009bb3ef1dd is now active with session: 18bc9466-4d6d-49cc-a978-37aa010ce4c3","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173842782Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124202024-02-06T22:20:55.436Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124212024-02-06T22:20:55.437Z,"time":"2024-02-06T22:20:54.17385817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594,"session_id":"18bc9466-4d6d-49cc-a978-37aa010ce4c3"}
124222024-02-06T22:20:55.437Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:54.173889466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594}
124232024-02-06T22:20:55.437ZLoading write count information from file "/tmp/test_repair_verify.data"
124242024-02-06T22:20:55.437ZRun Repair workload
124252024-02-06T22:20:55.437Z01/30 Write block 261 len 36864 data: 13 14 15 13 14 15 15 14 13
124262024-02-06T22:20:55.437Z02/30 Read block 529 len 8192
124272024-02-06T22:20:55.438Z03/30 Read block 497 len 36864
124282024-02-06T22:20:55.438Z04/30 Read block 372 len 24576
124292024-02-06T22:20:55.438Z05/30 Read block 244 len 28672
124302024-02-06T22:20:55.438Z06/30 Write block 409 len 20480 data: 12 12 13 10 9
124312024-02-06T22:20:55.438Z07/30 Flush
124322024-02-06T22:20:55.438Z08/30 Write block 564 len 24576 data: 9 10 10 12 13 11
124332024-02-06T22:20:55.438Z09/30 Flush
124342024-02-06T22:20:55.439Z10/30 Write block 313 len 24576 data: 13 13 13 12 11 10
124352024-02-06T22:20:55.439Z11/30 Write block 373 len 8192 data: 6 8
124362024-02-06T22:20:55.468Z12/30 Write block 335 len 40960 data: 12 13 12 12 8 8 7 10 7 8
124372024-02-06T22:20:55.468Z13/30 Read block 558 len 16384
124382024-02-06T22:20:55.468Z14/30 Read block 38 len 20480
124392024-02-06T22:20:55.468Z15/30 Write block 203 len 12288 data: 11 12 10
124402024-02-06T22:20:55.469Z16/30 Read block 200 len 24576
124412024-02-06T22:20:55.469Z17/30 Write block 188 len 28672 data: 11 10 8 9 8 8 11
124422024-02-06T22:20:55.469Z18/30 Read block 77 len 24576
124432024-02-06T22:20:55.469Z19/30 Read block 510 len 16384
124442024-02-06T22:20:55.469Z20/30 Read block 34 len 12288
124452024-02-06T22:20:55.469Z21/30 Read block 187 len 24576
124462024-02-06T22:20:55.470Z22/30 Write block 111 len 36864 data: 7 9 10 12 12 9 11 12 10
124472024-02-06T22:20:55.470Z23/30 Write block 578 len 8192 data: 13 12
124482024-02-06T22:20:55.470Z24/30 Write block 87 len 8192 data: 9 8
124492024-02-06T22:20:55.470Z25/30 Write block 48 len 4096 data: 8
124502024-02-06T22:20:55.470Z26/30 Read block 141 len 24576
124512024-02-06T22:20:55.470Z27/30 Write block 558 len 20480 data: 12 9 8 8 9
124522024-02-06T22:20:55.470Z28/30 Write block 546 len 12288 data: 10 8 7
124532024-02-06T22:20:55.471Z29/30 Read block 577 len 20480
124542024-02-06T22:20:55.471Z30/30 Write block 315 len 40960 data: 14 13 12 11 10 8 8 9 7 7
124552024-02-06T22:20:55.471Z----------------------------------------------------------------
124562024-02-06T22:20:55.471Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:21
124572024-02-06T22:20:55.471ZStates: Active Active Active
124582024-02-06T22:20:55.471ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
124592024-02-06T22:20:55.471Z 10 Acked 1009 Write 6 Done Done Sent false
124602024-02-06T22:20:55.472Z 11 Acked 1010 Write 2 Done Done Sent false
124612024-02-06T22:20:55.472Z 12 Acked 1011 Write 10 Done Done Sent false
124622024-02-06T22:20:55.472Z 13 Acked 1012 Read 4 Done Done Sent false
124632024-02-06T22:20:55.472Z 14 Acked 1013 Read 5 Done Done Sent false
124642024-02-06T22:20:55.472Z 15 Acked 1014 Write 3 Done Done Sent false
124652024-02-06T22:20:55.472Z 16 Acked 1015 Read 6 Done Done Sent false
124662024-02-06T22:20:55.473Z 17 Acked 1016 Write 7 Done Done Sent false
124672024-02-06T22:20:55.473Z 18 Acked 1017 Read 6 Done Done Sent false
124682024-02-06T22:20:55.473Z 19 Acked 1018 Read 4 Done Done Sent false
124692024-02-06T22:20:55.473Z 20 Acked 1019 Read 3 Done Done Sent false
124702024-02-06T22:20:55.473Z 21 Acked 1020 Read 6 Done Done Sent false
124712024-02-06T22:20:55.474Z 22 Acked 1021 Write 9 Done Done Sent false
124722024-02-06T22:20:55.474Z 23 Acked 1022 Write 2 Done Done Sent false
124732024-02-06T22:20:55.474Z 24 Acked 1023 Write 2 Done Done Sent false
124742024-02-06T22:20:55.474Z 25 Acked 1024 Write 1 Done Done Sent false
124752024-02-06T22:20:55.474Z 26 Acked 1025 Read 6 Done Done Sent false
124762024-02-06T22:20:55.474Z 27 Acked 1026 Write 5 Done Done Sent false
124772024-02-06T22:20:55.474Z 28 Acked 1027 Write 3 Done Done Sent false
124782024-02-06T22:20:55.475Z 29 Acked 1028 Read 5 Done Done Sent false
124792024-02-06T22:20:55.475Z 30 Acked 1029 Write 10 Sent Sent Sent false
124802024-02-06T22:20:55.475Z STATES DS:0 DS:1 DS:2 TOTAL
124812024-02-06T22:20:55.475Z New 0 0 0 0
124822024-02-06T22:20:55.475Z Sent 1 1 21 23
124832024-02-06T22:20:55.475Z Done 20 20 0 40
124842024-02-06T22:20:55.476Z Skipped 0 0 0 0
124852024-02-06T22:20:55.476Z Error 0 0 0 0
124862024-02-06T22:20:55.476ZLast Flush: 1008 1008 1008
124872024-02-06T22:20:55.476ZDownstairs last five completed: 1008 1007 1006 1005 1004
124882024-02-06T22:20:55.476ZUpstairs last five completed: 30 29 28 27 26
124892024-02-06T22:20:55.476Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:54.202464085Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1594}
124902024-02-06T22:20:55.476ZWrote out file "/tmp/test_repair_verify.data"
124912024-02-06T22:20:55.476Z
124922024-02-06T22:20:55.477ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
124932024-02-06T22:20:55.477ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124942024-02-06T22:20:55.477Z 0 000-019 108 108 108 204 204 204 F F F
124952024-02-06T22:20:55.477Z 1 020-039 102 102 102 193 193 193 F F F
124962024-02-06T22:20:55.477Z 2 040-059 108 108 108 204 204 204 T T F <---
124972024-02-06T22:20:55.477Z 3 060-079 109 109 109 205 205 205 F F F
124982024-02-06T22:20:55.477Z 4 080-099 110 110 110 207 207 207 T T F <---
124992024-02-06T22:20:55.478Z 5 100-119 110 110 110 207 207 207 T T F <---
125002024-02-06T22:20:55.478Z 6 120-139 112 112 112 213 213 213 F F F
125012024-02-06T22:20:55.478Z 7 140-159 111 111 111 211 211 211 F F F
125022024-02-06T22:20:55.478Z 8 160-179 112 112 112 212 212 212 F F F
125032024-02-06T22:20:55.478Z 9 180-199 112 112 112 212 212 212 T T F <---
125042024-02-06T22:20:55.478Z 10 200-219 110 110 110 210 210 210 T T F <---
125052024-02-06T22:20:55.479Z 11 220-239 110 110 110 207 207 207 F F F
125062024-02-06T22:20:55.479Z 12 240-259 110 110 110 206 206 206 F F F
125072024-02-06T22:20:55.479Z 13 260-279 114 114 114 215 215 215 F F F
125082024-02-06T22:20:55.479Z 14 280-299 108 108 108 204 204 204 F F F
125092024-02-06T22:20:55.479Z 15 300-319 112 112 112 213 213 213 T T F <---
125102024-02-06T22:20:55.479Z 16 320-339 112 112 112 213 213 213 T T F <---
125112024-02-06T22:20:55.480Z 17 340-359 112 112 112 213 213 213 T T F <---
125122024-02-06T22:20:55.480Z 18 360-379 108 108 108 204 204 204 T T F <---
125132024-02-06T22:20:55.480Z 19 380-399 108 108 108 204 204 204 F F F
125142024-02-06T22:20:55.480Z 20 400-419 114 114 114 215 215 215 F F F
125152024-02-06T22:20:55.480Z 21 420-439 111 111 111 211 211 211 F F F
125162024-02-06T22:20:55.480Z 22 440-459 112 112 112 213 213 213 F F F
125172024-02-06T22:20:55.480Z 23 460-479 112 112 112 213 213 213 F F F
125182024-02-06T22:20:55.481Z 24 480-499 113 113 113 214 214 214 F F F
125192024-02-06T22:20:55.481Z 25 500-519 112 112 112 212 212 212 F F F
125202024-02-06T22:20:55.481Z 26 520-539 104 104 104 199 199 199 F F F
125212024-02-06T22:20:55.481Z 27 540-559 111 111 111 211 211 211 T T F <---
125222024-02-06T22:20:55.481Z 28 560-579 114 114 114 216 216 216 T T F <---
125232024-02-06T22:20:55.481Z 29 580-599 112 112 112 213 213 213 F F F
125242024-02-06T22:20:55.481ZMax gen: 114, Max flush: 216
125252024-02-06T22:20:55.482ZError: Difference in extent metadata found!
125262024-02-06T22:20:55.482ZA Difference in extent metadata is expected here
125272024-02-06T22:20:55.482ZOn loop 57
125282024-02-06T22:20:55.482Z
125292024-02-06T22:20:55.482ZVerifying data now
125302024-02-06T22:20:59.603ZNo change after verify
125312024-02-06T22:20:59.603ZLoop: 57 Downstairs dump after verify (and repair):
125322024-02-06T22:20:59.619ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125332024-02-06T22:20:59.619Z 0 000-019 108 108 108 204 204 204 F F F
125342024-02-06T22:20:59.619Z 1 020-039 102 102 102 193 193 193 F F F
125352024-02-06T22:20:59.619Z 2 040-059 115 115 115 217 217 217 F F F
125362024-02-06T22:20:59.619Z 3 060-079 109 109 109 205 205 205 F F F
125372024-02-06T22:20:59.619Z 4 080-099 115 115 115 217 217 217 F F F
125382024-02-06T22:20:59.619Z 5 100-119 115 115 115 217 217 217 F F F
125392024-02-06T22:20:59.619Z 6 120-139 112 112 112 213 213 213 F F F
125402024-02-06T22:20:59.619Z 7 140-159 111 111 111 211 211 211 F F F
125412024-02-06T22:20:59.619Z 8 160-179 112 112 112 212 212 212 F F F
125422024-02-06T22:20:59.619Z 9 180-199 115 115 115 217 217 217 F F F
125432024-02-06T22:20:59.619Z 10 200-219 115 115 115 217 217 217 F F F
125442024-02-06T22:20:59.619Z 11 220-239 110 110 110 207 207 207 F F F
125452024-02-06T22:20:59.623Z 12 240-259 110 110 110 206 206 206 F F F
125462024-02-06T22:20:59.623Z 13 260-279 114 114 114 215 215 215 F F F
125472024-02-06T22:20:59.623Z 14 280-299 108 108 108 204 204 204 F F F
125482024-02-06T22:20:59.623Z 15 300-319 115 115 115 217 217 217 F F F
125492024-02-06T22:20:59.623Z 16 320-339 115 115 115 217 217 217 F F F
125502024-02-06T22:20:59.623Z 17 340-359 115 115 115 217 217 217 F F F
125512024-02-06T22:20:59.623Z 18 360-379 115 115 115 217 217 217 F F F
125522024-02-06T22:20:59.623Z 19 380-399 108 108 108 204 204 204 F F F
125532024-02-06T22:20:59.623Z 20 400-419 114 114 114 215 215 215 F F F
125542024-02-06T22:20:59.623Z 21 420-439 111 111 111 211 211 211 F F F
125552024-02-06T22:20:59.623Z 22 440-459 112 112 112 213 213 213 F F F
125562024-02-06T22:20:59.623Z 23 460-479 112 112 112 213 213 213 F F F
125572024-02-06T22:20:59.623Z 24 480-499 113 113 113 214 214 214 F F F
125582024-02-06T22:20:59.623Z 25 500-519 112 112 112 212 212 212 F F F
125592024-02-06T22:20:59.623Z 26 520-539 104 104 104 199 199 199 F F F
125602024-02-06T22:20:59.623Z 27 540-559 115 115 115 217 217 217 F F F
125612024-02-06T22:20:59.623Z 28 560-579 115 115 115 217 217 217 F F F
125622024-02-06T22:20:59.623Z 29 580-599 112 112 112 213 213 213 F F F
125632024-02-06T22:20:59.624ZMax gen: 115, Max flush: 217
125642024-02-06T22:20:59.624Z
125652024-02-06T22:20:59.624ZBegin loop 58
125662024-02-06T22:20:59.624ZDownstairs to restart: 0
125672024-02-06T22:20:59.641Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.839495506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602}
125682024-02-06T22:20:59.641Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.839728095Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602}
125692024-02-06T22:20:59.641Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.839745043Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602}
125702024-02-06T22:20:59.646Z{"msg":"Crucible fd701b66-e534-471a-a040-662c9b80cc6a has session id: 0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.839786577Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125712024-02-06T22:20:59.646ZCrucible runtime is spawned
125722024-02-06T22:20:59.647Z{"msg":"Upstairs opts: Upstairs UUID: fd701b66-e534-471a-a040-662c9b80cc6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.839870826Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125732024-02-06T22:20:59.647Z{"msg":"Crucible stats registered with UUID: fd701b66-e534-471a-a040-662c9b80cc6a","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.83991073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125742024-02-06T22:20:59.647Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.839932717Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602}
125752024-02-06T22:20:59.647Z{"msg":"fd701b66-e534-471a-a040-662c9b80cc6a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840049771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125762024-02-06T22:20:59.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840095535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125772024-02-06T22:20:59.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840116133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125782024-02-06T22:20:59.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840157727Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125792024-02-06T22:20:59.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840185453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125802024-02-06T22:20:59.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840269972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125812024-02-06T22:20:59.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840308107Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125822024-02-06T22:20:59.647Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840335993Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125832024-02-06T22:20:59.647Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840368199Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125842024-02-06T22:20:59.647Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840407793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125852024-02-06T22:20:59.647Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.84043277Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125862024-02-06T22:20:59.647Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840463146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125872024-02-06T22:20:59.647Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.840487363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125882024-02-06T22:20:59.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.841144874Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125892024-02-06T22:20:59.647Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.841195097Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125902024-02-06T22:20:59.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.841223923Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125912024-02-06T22:20:59.647Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.841257468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125922024-02-06T22:20:59.647Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.841843639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125932024-02-06T22:20:59.647Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.841906071Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125942024-02-06T22:20:59.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.842200761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125952024-02-06T22:20:59.648Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.842244195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125962024-02-06T22:20:59.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.842463265Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125972024-02-06T22:20:59.648Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.84250529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125982024-02-06T22:20:59.648Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.842529736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
125992024-02-06T22:20:59.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.842561052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126002024-02-06T22:20:59.648Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.84257913Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126012024-02-06T22:20:59.648Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.842600937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126022024-02-06T22:20:59.648Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.842829256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126032024-02-06T22:20:59.652Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843296453Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126042024-02-06T22:20:59.653Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843343666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126052024-02-06T22:20:59.653Z{"msg":"[0]R flush_numbers[0..12]: [204, 193, 217, 205, 217, 217, 213, 211, 212, 217, 217, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843382671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126062024-02-06T22:20:59.653Z{"msg":"[0]R generation[0..12]: [108, 102, 115, 109, 115, 115, 112, 111, 112, 115, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843419356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126072024-02-06T22:20:59.653Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843472879Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126082024-02-06T22:20:59.653Z{"msg":"[1]R flush_numbers[0..12]: [204, 193, 217, 205, 217, 217, 213, 211, 212, 217, 217, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843513104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126092024-02-06T22:20:59.653ZWait for a query_work_queue command to finish before sending IO
126102024-02-06T22:20:59.653Z{"msg":"[1]R generation[0..12]: [108, 102, 115, 109, 115, 115, 112, 111, 112, 115, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843554768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126112024-02-06T22:20:59.653Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843606181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126122024-02-06T22:20:59.653Z{"msg":"[2]R flush_numbers[0..12]: [204, 193, 217, 205, 217, 217, 213, 211, 212, 217, 217, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843643426Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126132024-02-06T22:20:59.653Z{"msg":"[2]R generation[0..12]: [108, 102, 115, 109, 115, 115, 112, 111, 112, 115, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843680671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126142024-02-06T22:20:59.653Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843720166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126152024-02-06T22:20:59.653Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843757171Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126162024-02-06T22:20:59.653Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843794645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126172024-02-06T22:20:59.653Z{"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843830761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126182024-02-06T22:20:59.653ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
126192024-02-06T22:20:59.653Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843867036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126202024-02-06T22:20:59.653Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843902221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126212024-02-06T22:20:59.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843917789Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126222024-02-06T22:20:59.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843938966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126232024-02-06T22:20:59.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843952814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126242024-02-06T22:20:59.653Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.843973721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126252024-02-06T22:20:59.653Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.8439868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126262024-02-06T22:20:59.653Z{"msg":"fd701b66-e534-471a-a040-662c9b80cc6a is now active with session: 0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.844022855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126272024-02-06T22:20:59.653Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.844036533Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602,"session_id":"0c609d5e-87bf-4eb2-80cd-c7ec8c6dd66d"}
126282024-02-06T22:20:59.654Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:20:58.84405615Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602}
126292024-02-06T22:20:59.654ZLoading write count information from file "/tmp/test_repair_verify.data"
126302024-02-06T22:20:59.654ZRun Repair workload
126312024-02-06T22:20:59.654Z01/30 Write block 170 len 32768 data: 12 9 11 14 14 13 11 10
126322024-02-06T22:20:59.654Z02/30 Write block 533 len 40960 data: 10 7 7 7 6 5 4 5 7 7
126332024-02-06T22:20:59.654Z03/30 Read block 98 len 24576
126342024-02-06T22:20:59.654Z04/30 Write block 302 len 24576 data: 11 11 9 11 9 7
126352024-02-06T22:20:59.654Z05/30 Read block 327 len 16384
126362024-02-06T22:20:59.657Z06/30 Flush
126372024-02-06T22:20:59.657Z07/30 Write block 446 len 40960 data: 6 6 8 7 7 7 6 6 8 8
126382024-02-06T22:20:59.657Z08/30 Flush
126392024-02-06T22:20:59.657Z09/30 Write block 44 len 32768 data: 8 8 9 8 9 12 13 16
126402024-02-06T22:20:59.660Z10/30 Read block 147 len 12288
126412024-02-06T22:20:59.660Z11/30 Write block 373 len 12288 data: 7 9 8
126422024-02-06T22:20:59.660Z12/30 Write block 28 len 4096 data: 8
126432024-02-06T22:20:59.663Z13/30 Write block 162 len 40960 data: 10 8 8 6 7 9 11 13 13 10
126442024-02-06T22:20:59.663Z14/30 Read block 432 len 24576
126452024-02-06T22:20:59.663Z15/30 Write block 143 len 28672 data: 7 8 9 11 12 14 15
126462024-02-06T22:20:59.668Z16/30 Read block 215 len 32768
126472024-02-06T22:20:59.668Z17/30 Read block 424 len 4096
126482024-02-06T22:20:59.668Z18/30 Write block 295 len 8192 data: 14 12
126492024-02-06T22:20:59.668Z19/30 Read block 232 len 24576
126502024-02-06T22:20:59.671Z20/30 Read block 463 len 24576
126512024-02-06T22:20:59.671Z21/30 Write block 583 len 16384 data: 6 8 10 10
126522024-02-06T22:20:59.671Z22/30 Read block 469 len 40960
126532024-02-06T22:20:59.671Z23/30 Read block 84 len 40960
126542024-02-06T22:20:59.671Z24/30 Write block 271 len 12288 data: 10 10 7
126552024-02-06T22:20:59.674Z25/30 Read block 527 len 32768
126562024-02-06T22:20:59.674Z26/30 Write block 351 len 16384 data: 17 18 16 15
126572024-02-06T22:20:59.674Z27/30 Write block 416 len 28672 data: 12 15 14 13 12 10 8
126582024-02-06T22:20:59.674Z28/30 Read block 566 len 20480
126592024-02-06T22:20:59.677Z29/30 Read block 491 len 32768
126602024-02-06T22:20:59.677Z30/30 Read block 80 len 4096
126612024-02-06T22:20:59.677Z----------------------------------------------------------------
126622024-02-06T22:20:59.677Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
126632024-02-06T22:20:59.677ZStates: Active Active Active
126642024-02-06T22:20:59.678ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
126652024-02-06T22:20:59.678Z 1 Acked 1000 Write 8 Done Done Done false
126662024-02-06T22:20:59.678Z 2 Acked 1001 Write 10 Done Done Done false
126672024-02-06T22:20:59.678Z 3 Acked 1002 Read 6 Done Done Done false
126682024-02-06T22:20:59.678Z 4 Acked 1003 Write 6 Sent Done Done false
126692024-02-06T22:20:59.678Z 5 Acked 1004 Read 4 Sent Done Done false
126702024-02-06T22:20:59.678Z 6 Acked 1005 Flush 0 Sent Done Done false
126712024-02-06T22:20:59.678Z 7 Acked 1006 Write 10 Sent Done Done false
126722024-02-06T22:20:59.678Z 8 Acked 1007 Flush 0 Sent Done Done false
126732024-02-06T22:20:59.678Z 9 Acked 1008 Write 8 Sent Done Done false
126742024-02-06T22:20:59.678Z 10 Acked 1009 Read 3 Sent Done Done false
126752024-02-06T22:20:59.678Z 11 Acked 1010 Write 3 Sent Done Done false
126762024-02-06T22:20:59.678Z 12 Acked 1011 Write 1 Sent Done Done false
126772024-02-06T22:20:59.678Z 13 Acked 1012 Write 10 Sent Done Done false
126782024-02-06T22:20:59.678Z 14 Acked 1013 Read 6 Sent Done Done false
126792024-02-06T22:20:59.678Z 15 Acked 1014 Write 7 Sent Done Done false
126802024-02-06T22:20:59.678Z 16 Acked 1015 Read 8 Sent Done Done false
126812024-02-06T22:20:59.678Z 17 Acked 1016 Read 1 Sent Done Done false
126822024-02-06T22:20:59.678Z 18 Acked 1017 Write 2 Sent Done Done false
126832024-02-06T22:20:59.678Z 19 Acked 1018 Read 6 Sent Done Done false
126842024-02-06T22:20:59.678Z 20 Acked 1019 Read 6 Sent Done Done false
126852024-02-06T22:20:59.678Z 21 Acked 1020 Write 4 Sent Done Done false
126862024-02-06T22:20:59.678Z 22 Acked 1021 Read 10 Sent Done Done false
126872024-02-06T22:20:59.678Z 23 Acked 1022 Read 10 Sent Done Done false
126882024-02-06T22:20:59.679Z 24 Acked 1023 Write 3 Sent Done Done false
126892024-02-06T22:20:59.679Z 25 Acked 1024 Read 8 Sent Done Done false
126902024-02-06T22:20:59.679Z 26 Acked 1025 Write 4 Sent Done Done false
126912024-02-06T22:20:59.679Z 27 Acked 1026 Write 7 Sent Done Done false
126922024-02-06T22:20:59.679Z 28 Acked 1027 Read 5 Sent Done Done false
126932024-02-06T22:20:59.679Z 29 Acked 1028 Read 8 Sent Done Done false
126942024-02-06T22:20:59.679Z 30 Acked 1029 Read 1 Sent Done Done false
126952024-02-06T22:20:59.679Z STATES DS:0 DS:1 DS:2 TOTAL
126962024-02-06T22:20:59.679Z New 0 0 0 0
126972024-02-06T22:20:59.679Z Sent 27 0 0 27
126982024-02-06T22:20:59.679Z Done 3 30 30 63
126992024-02-06T22:20:59.679Z Skipped 0 0 0 0
127002024-02-06T22:20:59.679Z Error 0 0 0 0
127012024-02-06T22:20:59.679ZLast Flush: 0 1007 1007
127022024-02-06T22:20:59.679ZDownstairs last five completed:
127032024-02-06T22:20:59.679ZUpstairs last five completed: 30 29 28 27 26
127042024-02-06T22:20:59.679Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:20:58.875559434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1602}
127052024-02-06T22:20:59.679ZWrote out file "/tmp/test_repair_verify.data"
127062024-02-06T22:20:59.682Z
127072024-02-06T22:20:59.688ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127082024-02-06T22:20:59.703ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127092024-02-06T22:20:59.703Z 0 000-019 108 108 108 204 204 204 F F F
127102024-02-06T22:20:59.703Z 1 020-039 102 102 102 193 193 193 F T T <---
127112024-02-06T22:20:59.703Z 2 040-059 115 115 115 217 217 217 F T T <---
127122024-02-06T22:20:59.704Z 3 060-079 109 109 109 205 205 205 F F F
127132024-02-06T22:20:59.704Z 4 080-099 115 115 115 217 217 217 F F F
127142024-02-06T22:20:59.704Z 5 100-119 115 115 115 217 217 217 F F F
127152024-02-06T22:20:59.707Z 6 120-139 112 112 112 213 213 213 F F F
127162024-02-06T22:20:59.707Z 7 140-159 111 111 111 211 211 211 F T T <---
127172024-02-06T22:20:59.707Z 8 160-179 112 116 116 212 218 218 T T T <---
127182024-02-06T22:20:59.707Z 9 180-199 115 115 115 217 217 217 F F F
127192024-02-06T22:20:59.707Z 10 200-219 115 115 115 217 217 217 F F F
127202024-02-06T22:20:59.707Z 11 220-239 110 110 110 207 207 207 F F F
127212024-02-06T22:20:59.707Z 12 240-259 110 110 110 206 206 206 F F F
127222024-02-06T22:20:59.707Z 13 260-279 114 114 114 215 215 215 F T T <---
127232024-02-06T22:20:59.707Z 14 280-299 108 108 108 204 204 204 F T T <---
127242024-02-06T22:20:59.707Z 15 300-319 115 116 116 217 218 218 F F F <---
127252024-02-06T22:20:59.707Z 16 320-339 115 115 115 217 217 217 F F F
127262024-02-06T22:20:59.707Z 17 340-359 115 115 115 217 217 217 F T T <---
127272024-02-06T22:20:59.707Z 18 360-379 115 115 115 217 217 217 F T T <---
127282024-02-06T22:20:59.707Z 19 380-399 108 108 108 204 204 204 F F F
127292024-02-06T22:20:59.707Z 20 400-419 114 114 114 215 215 215 F T T <---
127302024-02-06T22:20:59.708Z 21 420-439 111 111 111 211 211 211 F T T <---
127312024-02-06T22:20:59.708Z 22 440-459 112 116 116 213 219 219 F F F <---
127322024-02-06T22:20:59.708Z 23 460-479 112 112 112 213 213 213 F F F
127332024-02-06T22:20:59.708Z 24 480-499 113 113 113 214 214 214 F F F
127342024-02-06T22:20:59.708Z 25 500-519 112 112 112 212 212 212 F F F
127352024-02-06T22:20:59.708Z 26 520-539 104 116 116 199 218 218 T F F <---
127362024-02-06T22:20:59.708Z 27 540-559 115 116 116 217 218 218 T F F <---
127372024-02-06T22:20:59.708Z 28 560-579 115 115 115 217 217 217 F F F
127382024-02-06T22:20:59.708Z 29 580-599 112 112 112 213 213 213 F T T <---
127392024-02-06T22:20:59.708ZMax gen: 116, Max flush: 219
127402024-02-06T22:20:59.708ZError: Difference in extent metadata found!
127412024-02-06T22:20:59.708ZA Difference in extent metadata is expected here
127422024-02-06T22:20:59.708ZOn loop 58
127432024-02-06T22:20:59.708Z
127442024-02-06T22:20:59.711ZVerifying data now
127452024-02-06T22:21:05.447ZNo change after verify
127462024-02-06T22:21:05.448ZLoop: 58 Downstairs dump after verify (and repair):
127472024-02-06T22:21:05.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127482024-02-06T22:21:05.462Z 0 000-019 108 108 108 204 204 204 F F F
127492024-02-06T22:21:05.462Z 1 020-039 117 117 117 220 220 220 F F F
127502024-02-06T22:21:05.462Z 2 040-059 117 117 117 220 220 220 F F F
127512024-02-06T22:21:05.462Z 3 060-079 109 109 109 205 205 205 F F F
127522024-02-06T22:21:05.462Z 4 080-099 115 115 115 217 217 217 F F F
127532024-02-06T22:21:05.462Z 5 100-119 115 115 115 217 217 217 F F F
127542024-02-06T22:21:05.462Z 6 120-139 112 112 112 213 213 213 F F F
127552024-02-06T22:21:05.462Z 7 140-159 117 117 117 220 220 220 F F F
127562024-02-06T22:21:05.462Z 8 160-179 117 117 117 220 220 220 F F F
127572024-02-06T22:21:05.466Z 9 180-199 115 115 115 217 217 217 F F F
127582024-02-06T22:21:05.466Z 10 200-219 115 115 115 217 217 217 F F F
127592024-02-06T22:21:05.466Z 11 220-239 110 110 110 207 207 207 F F F
127602024-02-06T22:21:05.466Z 12 240-259 110 110 110 206 206 206 F F F
127612024-02-06T22:21:05.466Z 13 260-279 117 117 117 220 220 220 F F F
127622024-02-06T22:21:05.466Z 14 280-299 117 117 117 220 220 220 F F F
127632024-02-06T22:21:05.466Z 15 300-319 116 116 116 218 218 218 F F F
127642024-02-06T22:21:05.466Z 16 320-339 115 115 115 217 217 217 F F F
127652024-02-06T22:21:05.466Z 17 340-359 117 117 117 220 220 220 F F F
127662024-02-06T22:21:05.467Z 18 360-379 117 117 117 220 220 220 F F F
127672024-02-06T22:21:05.467Z 19 380-399 108 108 108 204 204 204 F F F
127682024-02-06T22:21:05.467Z 20 400-419 117 117 117 220 220 220 F F F
127692024-02-06T22:21:05.467Z 21 420-439 117 117 117 220 220 220 F F F
127702024-02-06T22:21:05.467Z 22 440-459 116 116 116 219 219 219 F F F
127712024-02-06T22:21:05.467Z 23 460-479 112 112 112 213 213 213 F F F
127722024-02-06T22:21:05.467Z 24 480-499 113 113 113 214 214 214 F F F
127732024-02-06T22:21:05.467Z 25 500-519 112 112 112 212 212 212 F F F
127742024-02-06T22:21:05.467Z 26 520-539 116 116 116 218 218 218 F F F
127752024-02-06T22:21:05.467Z 27 540-559 116 116 116 218 218 218 F F F
127762024-02-06T22:21:05.467Z 28 560-579 115 115 115 217 217 217 F F F
127772024-02-06T22:21:05.467Z 29 580-599 117 117 117 220 220 220 F F F
127782024-02-06T22:21:05.467ZMax gen: 117, Max flush: 220
127792024-02-06T22:21:05.467Z
127802024-02-06T22:21:05.467ZBegin loop 59
127812024-02-06T22:21:05.467ZDownstairs to restart: 1
127822024-02-06T22:21:05.484Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.681694613Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610}
127832024-02-06T22:21:05.484Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.681965896Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610}
127842024-02-06T22:21:05.490Z{Crucible runtime is spawned
127852024-02-06T22:21:05.490Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682107837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610}
127862024-02-06T22:21:05.490Z{"msg":"Crucible 85cbce6f-44e6-4cc6-a7ce-524138891a3b has session id: 07ed6f6d-ef07-4e75-8e7e-61abaaa2a404","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682154831Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127872024-02-06T22:21:05.490Z{"msg":"Upstairs opts: Upstairs UUID: 85cbce6f-44e6-4cc6-a7ce-524138891a3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682194166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127882024-02-06T22:21:05.490Z{"msg":"Crucible stats registered with UUID: 85cbce6f-44e6-4cc6-a7ce-524138891a3b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682229761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127892024-02-06T22:21:05.490Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682254777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610}
127902024-02-06T22:21:05.490Z{"msg":"85cbce6f-44e6-4cc6-a7ce-524138891a3b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682284973Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127912024-02-06T22:21:05.490Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682324238Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127922024-02-06T22:21:05.490Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682351024Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127932024-02-06T22:21:05.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682390569Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127942024-02-06T22:21:05.491Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682414296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127952024-02-06T22:21:05.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682436333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127962024-02-06T22:21:05.491Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682466789Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127972024-02-06T22:21:05.491Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682489696Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127982024-02-06T22:21:05.491Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682516732Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
127992024-02-06T22:21:05.491Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682539339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128002024-02-06T22:21:05.491Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682561976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128012024-02-06T22:21:05.491Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682585783Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128022024-02-06T22:21:05.491Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.682618758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128032024-02-06T22:21:05.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.683291617Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128042024-02-06T22:21:05.491Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.683330012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128052024-02-06T22:21:05.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.68334857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128062024-02-06T22:21:05.491Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.683396143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128072024-02-06T22:21:05.491Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.683960867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128082024-02-06T22:21:05.491Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.684165299Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128092024-02-06T22:21:05.491Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.684484246Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128102024-02-06T22:21:05.491Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.68453241Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128112024-02-06T22:21:05.491Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.684555407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128122024-02-06T22:21:05.491Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.684579114Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128132024-02-06T22:21:05.491Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.68460681Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128142024-02-06T22:21:05.491Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.684712805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128152024-02-06T22:21:05.491Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.68475088Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128162024-02-06T22:21:05.491Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.684770977Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128172024-02-06T22:21:05.492Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685071747Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128182024-02-06T22:21:05.499Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685532435Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128192024-02-06T22:21:05.499Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685573179Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128202024-02-06T22:21:05.499Z{"msg":"[0]R flush_numbers[0..12]: [204, 220, 220, 205, 217, 217, 213, 220, 220, 217, 217, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685590707Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128212024-02-06T22:21:05.499Z{"msg":"[0]R generation[0..12]: [108, 117, 117, 109, 115, 115, 112, 117, 117, 115, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685613364Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128222024-02-06T22:21:05.499Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685628502Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128232024-02-06T22:21:05.499Z{"msg":"[1]R flush_numbers[0..12]: [204, 220, 220, 205, 217, 217, 213, 220, 220, 217, 217, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685650809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128242024-02-06T22:21:05.499Z{"msg":"[1]R generation[0..12]: [108, 117, 117, 109, 115, 115, 112, 117, 117, 115, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685665507Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128252024-02-06T22:21:05.499Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685680045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128262024-02-06T22:21:05.499Z{"msg":"[2]R flush_numbers[0..12]: [204, 220, 220, 205, 217, 217, 213, 220, 220, 217, 217, 207]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685702812Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128272024-02-06T22:21:05.499Z{"msg":"[2]R generation[0..12]: [108, 117, 117, 109, 115, 115, 112, 117, 117, 115, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685717909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128282024-02-06T22:21:05.499Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685746086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128292024-02-06T22:21:05.499Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685766793ZWait for a query_work_queue command to finish before sending IO
128302024-02-06T22:21:05.499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128312024-02-06T22:21:05.499Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685796359Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128322024-02-06T22:21:05.499Z{"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685817586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128332024-02-06T22:21:05.499Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685831384Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128342024-02-06T22:21:05.499Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685852231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128352024-02-06T22:21:05.499Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.68586462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128362024-02-06T22:21:05.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685885427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128372024-02-06T22:21:05.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685899165Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128382024-02-06T22:21:05.500Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685913153Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128392024-02-06T22:21:05.500Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685932071Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128402024-02-06T22:21:05.500Z{"msg":"85cbce6f-44e6-4cc6-a7ce-524138891a3b is now active with session: 07ed6f6d-ef07-4e75-8e7e-61abaaa2a404","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685961827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128412024-02-06T22:21:05.500Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.685979544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610,"session_id":"07ed6f6d-ef07-4e75-8e7e-61abaaa2a404"}
128422024-02-06T22:21:05.500Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:04.686020859Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610}
128432024-02-06T22:21:05.500ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128442024-02-06T22:21:05.500ZLoading write count information from file "/tmp/test_repair_verify.data"
128452024-02-06T22:21:05.500ZRun Repair workload
128462024-02-06T22:21:05.500Z01/30 Write block 585 len 8192 data: 11 11
128472024-02-06T22:21:05.500Z02/30 Read block 64 len 36864
128482024-02-06T22:21:05.500Z03/30 Flush
128492024-02-06T22:21:05.500Z04/30 Write block 378 len 36864 data: 13 11 11 12 10 10 9 11 12
128502024-02-06T22:21:05.500Z05/30 Write block 3 len 4096 data: 8
128512024-02-06T22:21:05.500Z06/30 Write block 411 len 24576 data: 14 11 10 9 12 13
128522024-02-06T22:21:05.500Z07/30 Write block 239 len 24576 data: 6 5 6 7 6 6
128532024-02-06T22:21:05.504Z08/30 Read block 95 len 12288
128542024-02-06T22:21:05.504Z09/30 Read block 108 len 20480
128552024-02-06T22:21:05.504Z10/30 Write block 214 len 4096 data: 12
128562024-02-06T22:21:05.504Z11/30 Flush
128572024-02-06T22:21:05.504Z12/30 Write block 262 len 24576 data: 15 16 14 15 16 16
128582024-02-06T22:21:05.504Z13/30 Write block 505 len 24576 data: 11 12 12 11 10 10
128592024-02-06T22:21:05.504Z14/30 Read block 429 len 20480
128602024-02-06T22:21:05.504Z15/30 Read block 142 len 8192
128612024-02-06T22:21:05.508Z16/30 Flush
128622024-02-06T22:21:05.509Z17/30 Write block 517 len 40960 data: 11 9 4 6 6 6 5 5 5 8
128632024-02-06T22:21:05.509Z18/30 Read block 587 len 40960
128642024-02-06T22:21:05.512Z19/30 Write block 178 len 16384 data: 12 16 15 15
128652024-02-06T22:21:05.512Z20/30 Read block 517 len 32768
128662024-02-06T22:21:05.512Z21/30 Write block 514 len 4096 data: 12
128672024-02-06T22:21:05.512Z22/30 Read block 457 len 20480
128682024-02-06T22:21:05.512Z23/30 Read block 331 len 32768
128692024-02-06T22:21:05.516Z24/30 Read block 164 len 4096
128702024-02-06T22:21:05.516Z25/30 Write block 306 len 16384 data: 10 8 9 9
128712024-02-06T22:21:05.516Z26/30 Read block 220 len 20480
128722024-02-06T22:21:05.516Z27/30 Write block 442 len 20480 data: 9 7 7 7 7
128732024-02-06T22:21:05.521Z28/30 Write block 426 len 32768 data: 11 9 10 9 8 4 5 6
128742024-02-06T22:21:05.521Z29/30 Read block 350 len 20480
128752024-02-06T22:21:05.521Z30/30 Write block 572 len 24576 data: 15 14 15 12 12 13
128762024-02-06T22:21:05.521Z----------------------------------------------------------------
128772024-02-06T22:21:05.521Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:27
128782024-02-06T22:21:05.521ZStates: Active Active Active
128792024-02-06T22:21:05.521ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
128802024-02-06T22:21:05.521Z 4 Acked 1003 Write 9 Done Done Done false
128812024-02-06T22:21:05.522Z 5 Acked 1004 Write 1 Done Done Done false
128822024-02-06T22:21:05.522Z 6 Acked 1005 Write 6 Done Done Done false
128832024-02-06T22:21:05.522Z 7 Acked 1006 Write 6 Done Done Done false
128842024-02-06T22:21:05.522Z 8 Acked 1007 Read 3 Done Sent Done false
128852024-02-06T22:21:05.522Z 9 Acked 1008 Read 5 Done Sent Done false
128862024-02-06T22:21:05.522Z 10 Acked 1009 Write 1 Done Sent Done false
128872024-02-06T22:21:05.522Z 11 Acked 1010 Flush 0 Done Sent Done false
128882024-02-06T22:21:05.522Z 12 Acked 1011 Write 6 Done Sent Done false
128892024-02-06T22:21:05.522Z 13 Acked 1012 Write 6 Done Sent Done false
128902024-02-06T22:21:05.522Z 14 Acked 1013 Read 5 Done Sent Done false
128912024-02-06T22:21:05.522Z 15 Acked 1014 Read 2 Done Sent Done false
128922024-02-06T22:21:05.522Z 16 Acked 1015 Flush 0 Done Sent Done false
128932024-02-06T22:21:05.522Z 17 Acked 1016 Write 10 Done Sent Done false
128942024-02-06T22:21:05.522Z 18 Acked 1017 Read 10 Done Sent Done false
128952024-02-06T22:21:05.522Z 19 Acked 1018 Write 4 Done Sent Done false
128962024-02-06T22:21:05.522Z 20 Acked 1019 Read 8 Done Sent Done false
128972024-02-06T22:21:05.522Z 21 Acked 1020 Write 1 Done Sent Done false
128982024-02-06T22:21:05.522Z 22 Acked 1021 Read 5 Done Sent Done false
128992024-02-06T22:21:05.522Z 23 Acked 1022 Read 8 Done Sent Done false
129002024-02-06T22:21:05.522Z 24 Acked 1023 Read 1 Done Sent Done false
129012024-02-06T22:21:05.522Z 25 Acked 1024 Write 4 Done Sent Done false
129022024-02-06T22:21:05.522Z 26 Acked 1025 Read 5 Done Sent Done false
129032024-02-06T22:21:05.522Z 27 Acked 1026 Write 5 Done Sent Done false
129042024-02-06T22:21:05.522Z 28 Acked 1027 Write 8 Done Sent Done false
129052024-02-06T22:21:05.522Z 29 Acked 1028 Read 5 Done Sent Done false
129062024-02-06T22:21:05.523Z 30 Acked 1029 Write 6 Sent Sent Sent false
129072024-02-06T22:21:05.523Z STATES DS:0 DS:1 DS:2 TOTAL
129082024-02-06T22:21:05.523Z New 0 0 0 0
129092024-02-06T22:21:05.523Z Sent 1 23 1 25
129102024-02-06T22:21:05.523Z Done 26 4 26 56
129112024-02-06T22:21:05.523Z Skipped 0 0 0 0
129122024-02-06T22:21:05.523Z Error 0 0 0 0
129132024-02-06T22:21:05.523ZLast Flush: 1015 1002 1015
129142024-02-06T22:21:05.523ZDownstairs last five completed: 1002 1001 1000
129152024-02-06T22:21:05.523ZUpstairs last five completed: 30 29 28 27 26
129162024-02-06T22:21:05.523Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:04.716605586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1610}
129172024-02-06T22:21:05.523ZWrote out file "/tmp/test_repair_verify.data"
129182024-02-06T22:21:05.526Z
129192024-02-06T22:21:05.526ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
129202024-02-06T22:21:05.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129212024-02-06T22:21:05.545Z 0 000-019 118 108 118 222 204 222 F T F <---
129222024-02-06T22:21:05.545Z 1 020-039 117 117 117 220 220 220 F F F
129232024-02-06T22:21:05.545Z 2 040-059 117 117 117 220 220 220 F F F
129242024-02-06T22:21:05.546Z 3 060-079 109 109 109 205 205 205 F F F
129252024-02-06T22:21:05.546Z 4 080-099 115 115 115 217 217 217 F F F
129262024-02-06T22:21:05.546Z 5 100-119 115 115 115 217 217 217 F F F
129272024-02-06T22:21:05.546Z 6 120-139 112 112 112 213 213 213 F F F
129282024-02-06T22:21:05.546Z 7 140-159 117 117 117 220 220 220 F F F
129292024-02-06T22:21:05.546Z 8 160-179 117 117 117 220 220 220 T F T <---
129302024-02-06T22:21:05.546Z 9 180-199 115 115 115 217 217 217 T F T <---
129312024-02-06T22:21:05.546Z 10 200-219 118 115 118 222 217 222 F F F <---
129322024-02-06T22:21:05.553Z 11 220-239 118 110 118 222 207 222 F T F <---
129332024-02-06T22:21:05.553Z 12 240-259 118 110 118 222 206 222 F T F <---
129342024-02-06T22:21:05.553Z 13 260-279 118 117 118 223 220 223 F F F <---
129352024-02-06T22:21:05.553Z 14 280-299 117 117 117 220 220 220 F F F
129362024-02-06T22:21:05.553Z 15 300-319 116 116 116 218 218 218 T F T <---
129372024-02-06T22:21:05.553Z 16 320-339 115 115 115 217 217 217 F F F
129382024-02-06T22:21:05.554Z 17 340-359 117 117 117 220 220 220 F F F
129392024-02-06T22:21:05.554Z 18 360-379 118 117 118 222 220 222 F T F <---
129402024-02-06T22:21:05.554Z 19 380-399 118 108 118 222 204 222 F T F <---
129412024-02-06T22:21:05.554Z 20 400-419 118 117 118 222 220 222 F T F <---
129422024-02-06T22:21:05.554Z 21 420-439 117 117 117 220 220 220 T F T <---
129432024-02-06T22:21:05.554Z 22 440-459 116 116 116 219 219 219 T F T <---
129442024-02-06T22:21:05.554Z 23 460-479 112 112 112 213 213 213 F F F
129452024-02-06T22:21:05.554Z 24 480-499 113 113 113 214 214 214 F F F
129462024-02-06T22:21:05.554Z 25 500-519 118 112 118 223 212 223 T F T <---
129472024-02-06T22:21:05.554Z 26 520-539 116 116 116 218 218 218 T F T <---
129482024-02-06T22:21:05.554Z 27 540-559 116 116 116 218 218 218 F F F
129492024-02-06T22:21:05.554Z 28 560-579 115 115 115 217 217 217 T F T <---
129502024-02-06T22:21:05.554Z 29 580-599 118 118 118 221 221 221 F F F
129512024-02-06T22:21:05.554ZMax gen: 118, Max flush: 223
129522024-02-06T22:21:05.554ZError: Difference in extent metadata found!
129532024-02-06T22:21:05.554ZA Difference in extent metadata is expected here
129542024-02-06T22:21:05.554ZOn loop 59
129552024-02-06T22:21:05.554Z
129562024-02-06T22:21:05.557ZVerifying data now
129572024-02-06T22:21:11.326ZNo change after verify
129582024-02-06T22:21:11.326ZLoop: 59 Downstairs dump after verify (and repair):
129592024-02-06T22:21:11.354ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129602024-02-06T22:21:11.354Z 0 000-019 118 118 118 222 222 222 F F F
129612024-02-06T22:21:11.354Z 1 020-039 117 117 117 220 220 220 F F F
129622024-02-06T22:21:11.354Z 2 040-059 117 117 117 220 220 220 F F F
129632024-02-06T22:21:11.354Z 3 060-079 109 109 109 205 205 205 F F F
129642024-02-06T22:21:11.354Z 4 080-099 115 115 115 217 217 217 F F F
129652024-02-06T22:21:11.355Z 5 100-119 115 115 115 217 217 217 F F F
129662024-02-06T22:21:11.355Z 6 120-139 112 112 112 213 213 213 F F F
129672024-02-06T22:21:11.355Z 7 140-159 117 117 117 220 220 220 F F F
129682024-02-06T22:21:11.355Z 8 160-179 119 119 119 224 224 224 F F F
129692024-02-06T22:21:11.355Z 9 180-199 119 119 119 224 224 224 F F F
129702024-02-06T22:21:11.355Z 10 200-219 118 118 118 222 222 222 F F F
129712024-02-06T22:21:11.355Z 11 220-239 118 118 118 222 222 222 F F F
129722024-02-06T22:21:11.355Z 12 240-259 118 118 118 222 222 222 F F F
129732024-02-06T22:21:11.380Z 13 260-279 118 118 118 223 223 223 F F F
129742024-02-06T22:21:11.380Z 14 280-299 117 117 117 220 220 220 F F F
129752024-02-06T22:21:11.380Z 15 300-319 119 119 119 224 224 224 F F F
129762024-02-06T22:21:11.381Z 16 320-339 115 115 115 217 217 217 F F F
129772024-02-06T22:21:11.381Z 17 340-359 117 117 117 220 220 220 F F F
129782024-02-06T22:21:11.381Z 18 360-379 118 118 118 222 222 222 F F F
129792024-02-06T22:21:11.381Z 19 380-399 118 118 118 222 222 222 F F F
129802024-02-06T22:21:11.381Z 20 400-419 118 118 118 222 222 222 F F F
129812024-02-06T22:21:11.381Z 21 420-439 119 119 119 224 224 224 F F F
129822024-02-06T22:21:11.381Z 22 440-459 119 119 119 224 224 224 F F F
129832024-02-06T22:21:11.382Z 23 460-479 112 112 112 213 213 213 F F F
129842024-02-06T22:21:11.382Z 24 480-499 113 113 113 214 214 214 F F F
129852024-02-06T22:21:11.382Z 25 500-519 119 119 119 224 224 224 F F F
129862024-02-06T22:21:11.382Z 26 520-539 119 119 119 224 224 224 F F F
129872024-02-06T22:21:11.382Z 27 540-559 116 116 116 218 218 218 F F F
129882024-02-06T22:21:11.382Z 28 560-579 119 119 119 224 224 224 F F F
129892024-02-06T22:21:11.382Z 29 580-599 118 118 118 221 221 221 F F F
129902024-02-06T22:21:11.383ZMax gen: 119, Max flush: 224
129912024-02-06T22:21:11.383Z
129922024-02-06T22:21:11.383ZBegin loop 60
129932024-02-06T22:21:11.383ZDownstairs to restart: 2
129942024-02-06T22:21:11.383Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.554709467Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618}
129952024-02-06T22:21:11.383Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.554959353Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618}
129962024-02-06T22:21:11.383Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.554987259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618}
129972024-02-06T22:21:11.384Z{"msg":"Crucible 26a4eb2a-68d8-4bc1-b8e3-b2bec8196bf3 has session id: 4eb5d1df-683e-47b6-862f-bbad6eaab4e9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555008157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
129982024-02-06T22:21:11.384Z{"msg":"Upstairs opts: Upstairs UUID: 26a4eb2a-68d8-4bc1-b8e3-b2bec8196bf3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0Crucible runtime is spawned
129992024-02-06T22:21:11.384Z,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555050941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130002024-02-06T22:21:11.384Z{"msg":"Crucible stats registered with UUID: 26a4eb2a-68d8-4bc1-b8e3-b2bec8196bf3","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555076147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130012024-02-06T22:21:11.384Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555115722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618}
130022024-02-06T22:21:11.384Z{"msg":"26a4eb2a-68d8-4bc1-b8e3-b2bec8196bf3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555254213Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130032024-02-06T22:21:11.384Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.55528303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130042024-02-06T22:21:11.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555306166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130052024-02-06T22:21:11.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555341531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130062024-02-06T22:21:11.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555371887Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130072024-02-06T22:21:11.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555411082Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130082024-02-06T22:21:11.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555438549Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130092024-02-06T22:21:11.386Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555487132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130102024-02-06T22:21:11.386Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555531346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130112024-02-06T22:21:11.386Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555556982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130122024-02-06T22:21:11.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555581399Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130132024-02-06T22:21:11.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555626823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130142024-02-06T22:21:11.386Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:10.555812388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130152024-02-06T22:21:11.386Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:10.555868051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130162024-02-06T22:21:11.387Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:10.555904515Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130172024-02-06T22:21:11.387Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555926503Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130182024-02-06T22:21:11.387Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555949949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130192024-02-06T22:21:11.387Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.555976426Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130202024-02-06T22:21:11.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.556356434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130212024-02-06T22:21:11.387Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.556404688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130222024-02-06T22:21:11.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.556446252Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130232024-02-06T22:21:11.388Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.556475598Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130242024-02-06T22:21:11.388Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.556994658Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130252024-02-06T22:21:11.388Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.557152807Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130262024-02-06T22:21:11.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.557513678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130272024-02-06T22:21:11.388Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.557580499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130282024-02-06T22:21:11.388Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.557625643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130292024-02-06T22:21:11.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.55764606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130302024-02-06T22:21:11.389Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.557668837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130312024-02-06T22:21:11.389Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:10.557683245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130322024-02-06T22:21:12.367Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.556607938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130332024-02-06T22:21:12.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.556766186Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130342024-02-06T22:21:12.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.557569578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130352024-02-06T22:21:12.376Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.557613052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130362024-02-06T22:21:12.376Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558085758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130372024-02-06T22:21:12.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558598339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130382024-02-06T22:21:12.377Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558636434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130392024-02-06T22:21:12.377Z{"msg":"[0]R flush_numbers[0..12]: [222, 220, 220, 205, 217, 217, 213, 220, 224, 224, 222, 222]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558654561Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130402024-02-06T22:21:12.377Z{"msg":"[0]R generation[0..12]: [118, 117, 117, 109, 115, 115, 112, 117, 119, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558669429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130412024-02-06T22:21:12.377Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558698585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130422024-02-06T22:21:12.377Z{"msg":"[1]R flush_numbers[0..12]: [222, 220, 220, 205, 217, 217, 213, 220, 224, 224, 222, 222]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558716023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130432024-02-06T22:21:12.378Z{"msg":"[1]R generation[0..12]: [118, 117, 117, 109, 115, 115, 112, 117, 119, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558731641Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130442024-02-06T22:21:12.378Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558753898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130452024-02-06T22:21:12.378Z{"msg":"[2]R flush_numbers[0..12]: [222, 220, 220, 205, 217, 217, 213, 220, 224, 224, 222, 222]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558769506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130462024-02-06T22:21:12.378Z{"msg":"[2]R generation[0..12]: [118, 117, 117, 109, 115, 115, 112, 117, 119, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558790723Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130472024-02-06T22:21:12.378Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558805421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130482024-02-06T22:21:12.379Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
130492024-02-06T22:21:12.379Z,"time":"2024-02-06T22:21:11.558826818Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130502024-02-06T22:21:12.379Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558843936Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130512024-02-06T22:21:12.379Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558864663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130522024-02-06T22:21:12.379Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558878671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130532024-02-06T22:21:12.380Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558892389Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130542024-02-06T22:21:12.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558919086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130552024-02-06T22:21:12.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558949291Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130562024-02-06T22:21:12.380Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.558973768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130572024-02-06T22:21:12.380Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.559003964Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130582024-02-06T22:21:12.380Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.55903521Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130592024-02-06T22:21:12.380Z{"msg":"26a4eb2a-68d8-4bc1-b8e3-b2bec8196bf3 is now active with session: 4eb5d1df-683e-47b6-862f-bbad6eaab4e9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.559059667Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130602024-02-06T22:21:12.381Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.559074545Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618,"session_id":"4eb5d1df-683e-47b6-862f-bbad6eaab4e9"}
130612024-02-06T22:21:12.381Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:11.559087503Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618}
130622024-02-06T22:21:12.381ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
130632024-02-06T22:21:12.381ZLoading write count information from file "/tmp/test_repair_verify.data"
130642024-02-06T22:21:12.381ZRun Repair workload
130652024-02-06T22:21:12.381Z01/30 Write block 511 len 20480 data: 9 8 12 13 11
130662024-02-06T22:21:12.381Z02/30 Read block 145 len 32768
130672024-02-06T22:21:12.382Z03/30 Write block 337 len 40960 data: 13 13 9 9 8 11 8 9 9 11
130682024-02-06T22:21:12.382Z04/30 Read block 401 len 40960
130692024-02-06T22:21:12.382Z05/30 Read block 591 len 8192
130702024-02-06T22:21:12.382Z06/30 Read block 298 len 12288
130712024-02-06T22:21:12.382Z07/30 Write block 245 len 32768 data: 7 7 12 12 14 11 8 7
130722024-02-06T22:21:12.382Z08/30 Write block 33 len 24576 data: 5 5 4 5 6 6
130732024-02-06T22:21:12.394Z09/30 Write block 183 len 12288 data: 15 17 16
130742024-02-06T22:21:12.394Z10/30 Write block 298 len 16384 data: 8 9 11 12
130752024-02-06T22:21:12.395Z11/30 Read block 553 len 24576
130762024-02-06T22:21:12.395Z12/30 Write block 465 len 36864 data: 6 5 9 9 8 8 9 10 8
130772024-02-06T22:21:12.395Z13/30 Read block 58 len 8192
130782024-02-06T22:21:12.395Z14/30 Write block 18 len 24576 data: 7 6 5 6 8 8
130792024-02-06T22:21:12.396Z15/30 Write block 342 len 12288 data: 12 9 10
130802024-02-06T22:21:12.396Z16/30 Write block 40 len 32768 data: 12 10 9 9 9 9 10 9
130812024-02-06T22:21:12.396Z17/30 Write block 115 len 12288 data: 13 10 12
130822024-02-06T22:21:12.396Z18/30 Read block 537 len 36864
130832024-02-06T22:21:12.396Z19/30 Read block 399 len 16384
130842024-02-06T22:21:12.397Z20/30 Read block 419 len 16384
130852024-02-06T22:21:12.397Z21/30 Read block 211 len 20480
130862024-02-06T22:21:12.397Z22/30 Write block 149 len 16384 data: 16 12 10 10
130872024-02-06T22:21:12.397Z23/30 Read block 173 len 20480
130882024-02-06T22:21:12.397Z24/30 Write block 106 len 24576 data: 7 9 12 8 9 8
130892024-02-06T22:21:12.398Z25/30 Write block 339 len 8192 data: 10 10
130902024-02-06T22:21:12.398Z26/30 Write block 370 len 16384 data: 9 8 6 8
130912024-02-06T22:21:12.398Z27/30 Read block 122 len 24576
130922024-02-06T22:21:12.398Z28/30 Read block 113 len 24576
130932024-02-06T22:21:12.399Z29/30 Write block 464 len 20480 data: 8 7 6 10 10
130942024-02-06T22:21:12.411Z30/30 Read block 260 len 28672
130952024-02-06T22:21:12.411Z----------------------------------------------------------------
130962024-02-06T22:21:12.411Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
130972024-02-06T22:21:12.411ZStates: Active Active Active
130982024-02-06T22:21:12.411ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
130992024-02-06T22:21:12.411Z 1 Acked 1000 Write 5 Done Done Done false
131002024-02-06T22:21:12.412Z 2 Acked 1001 Read 8 Done Done Done false
131012024-02-06T22:21:12.412Z 3 Acked 1002 Write 10 Done Done Sent false
131022024-02-06T22:21:12.412Z 4 Acked 1003 Read 10 Done Done Sent false
131032024-02-06T22:21:12.412Z 5 Acked 1004 Read 2 Done Done Sent false
131042024-02-06T22:21:12.412Z 6 Acked 1005 Read 3 Done Done Sent false
131052024-02-06T22:21:12.412Z 7 Acked 1006 Write 8 Done Done Sent false
131062024-02-06T22:21:12.412Z 8 Acked 1007 Write 6 Done Done Sent false
131072024-02-06T22:21:12.413Z 9 Acked 1008 Write 3 Done Done Sent false
131082024-02-06T22:21:12.413Z 10 Acked 1009 Write 4 Done Done Sent false
131092024-02-06T22:21:12.413Z 11 Acked 1010 Read 6 Done Done Sent false
131102024-02-06T22:21:12.413Z 12 Acked 1011 Write 9 Done Done Sent false
131112024-02-06T22:21:12.413Z 13 Acked 1012 Read 2 Done Done Sent false
131122024-02-06T22:21:12.413Z 14 Acked 1013 Write 6 Done Done Sent false
131132024-02-06T22:21:12.413Z 15 Acked 1014 Write 3 Done Done Sent false
131142024-02-06T22:21:12.413Z 16 Acked 1015 Write 8 Done Done Sent false
131152024-02-06T22:21:12.413Z 17 Acked 1016 Write 3 Done Done Sent false
131162024-02-06T22:21:12.414Z 18 Acked 1017 Read 9 Done Done Sent false
131172024-02-06T22:21:12.414Z 19 Acked 1018 Read 4 Done Done Sent false
131182024-02-06T22:21:12.414Z 20 Acked 1019 Read 4 Done Done Sent false
131192024-02-06T22:21:12.414Z 21 Acked 1020 Read 5 Done Done Sent false
131202024-02-06T22:21:12.414Z 22 Acked 1021 Write 4 Done Done Sent false
131212024-02-06T22:21:12.414Z 23 Acked 1022 Read 5 Done Done Sent false
131222024-02-06T22:21:12.414Z 24 Acked 1023 Write 6 Done Done Sent false
131232024-02-06T22:21:12.415Z 25 Acked 1024 Write 2 Done Done Sent false
131242024-02-06T22:21:12.415Z 26 Acked 1025 Write 4 Done Done Sent false
131252024-02-06T22:21:12.415Z 27 Acked 1026 Read 6 Done Done Sent false
131262024-02-06T22:21:12.415Z 28 Acked 1027 Read 6 Done Done Sent false
131272024-02-06T22:21:12.415Z 29 Acked 1028 Write 5 Done Done Sent false
131282024-02-06T22:21:12.415Z 30 Acked 1029 Read 7 Sent Done Sent false
131292024-02-06T22:21:12.415Z STATES DS:0 DS:1 DS:2 TOTAL
131302024-02-06T22:21:12.416Z New 0 0 0 0
131312024-02-06T22:21:12.416Z Sent 1 0 28 29
131322024-02-06T22:21:12.416Z Done 29 30 2 61
131332024-02-06T22:21:12.417Z Skipped 0 0 0 0
131342024-02-06T22:21:12.417Z Error 0 0 0 0
131352024-02-06T22:21:12.417ZLast Flush: 0 0 0
131362024-02-06T22:21:12.417ZDownstairs last five completed:
131372024-02-06T22:21:12.417ZUpstairs last five completed: 30 29 28 27 26
131382024-02-06T22:21:12.418Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:11.586225726Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1618}
131392024-02-06T22:21:12.418ZWrote out file "/tmp/test_repair_verify.data"
131402024-02-06T22:21:12.418Z
131412024-02-06T22:21:12.418ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131422024-02-06T22:21:12.426ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131432024-02-06T22:21:12.427Z 0 000-019 118 118 118 222 222 222 T T F <---
131442024-02-06T22:21:12.427Z 1 020-039 117 117 117 220 220 220 T T F <---
131452024-02-06T22:21:12.427Z 2 040-059 117 117 117 220 220 220 T T F <---
131462024-02-06T22:21:12.427Z 3 060-079 109 109 109 205 205 205 F F F
131472024-02-06T22:21:12.427Z 4 080-099 115 115 115 217 217 217 F F F
131482024-02-06T22:21:12.427Z 5 100-119 115 115 115 217 217 217 T T F <---
131492024-02-06T22:21:12.427Z 6 120-139 112 112 112 213 213 213 F F F
131502024-02-06T22:21:12.427Z 7 140-159 117 117 117 220 220 220 T T F <---
131512024-02-06T22:21:12.428Z 8 160-179 119 119 119 224 224 224 F F F
131522024-02-06T22:21:12.428Z 9 180-199 119 119 119 224 224 224 T T F <---
131532024-02-06T22:21:12.428Z 10 200-219 118 118 118 222 222 222 F F F
131542024-02-06T22:21:12.428Z 11 220-239 118 118 118 222 222 222 F F F
131552024-02-06T22:21:12.428Z 12 240-259 118 118 118 222 222 222 T T F <---
131562024-02-06T22:21:12.428Z 13 260-279 118 118 118 223 223 223 F F F
131572024-02-06T22:21:12.428Z 14 280-299 117 117 117 220 220 220 T T F <---
131582024-02-06T22:21:12.428Z 15 300-319 119 119 119 224 224 224 T T F <---
131592024-02-06T22:21:12.428Z 16 320-339 115 115 115 217 217 217 T T F <---
131602024-02-06T22:21:12.428Z 17 340-359 117 117 117 220 220 220 T T F <---
131612024-02-06T22:21:12.429Z 18 360-379 118 118 118 222 222 222 T T F <---
131622024-02-06T22:21:12.429Z 19 380-399 118 118 118 222 222 222 F F F
131632024-02-06T22:21:12.429Z 20 400-419 118 118 118 222 222 222 F F F
131642024-02-06T22:21:12.429Z 21 420-439 119 119 119 224 224 224 F F F
131652024-02-06T22:21:12.429Z 22 440-459 119 119 119 224 224 224 F F F
131662024-02-06T22:21:12.429Z 23 460-479 112 112 112 213 213 213 T T F <---
131672024-02-06T22:21:12.429Z 24 480-499 113 113 113 214 214 214 F F F
131682024-02-06T22:21:12.429Z 25 500-519 119 119 119 224 224 224 T T T
131692024-02-06T22:21:12.430Z 26 520-539 119 119 119 224 224 224 F F F
131702024-02-06T22:21:12.430Z 27 540-559 116 116 116 218 218 218 F F F
131712024-02-06T22:21:12.430Z 28 560-579 119 119 119 224 224 224 F F F
131722024-02-06T22:21:12.430Z 29 580-599 118 118 118 221 221 221 F F F
131732024-02-06T22:21:12.430ZMax gen: 119, Max flush: 224
131742024-02-06T22:21:12.430ZError: Difference in extent metadata found!
131752024-02-06T22:21:12.430ZA Difference in extent metadata is expected here
131762024-02-06T22:21:12.430ZOn loop 60
131772024-02-06T22:21:12.430Z
131782024-02-06T22:21:12.440ZVerifying data now
131792024-02-06T22:21:17.136ZNo change after verify
131802024-02-06T22:21:17.136ZLoop: 60 Downstairs dump after verify (and repair):
131812024-02-06T22:21:17.151ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131822024-02-06T22:21:17.151Z 0 000-019 120 120 120 225 225 225 F F F
131832024-02-06T22:21:17.152Z 1 020-039 120 120 120 225 225 225 F F F
131842024-02-06T22:21:17.152Z 2 040-059 120 120 120 225 225 225 F F F
131852024-02-06T22:21:17.152Z 3 060-079 109 109 109 205 205 205 F F F
131862024-02-06T22:21:17.152Z 4 080-099 115 115 115 217 217 217 F F F
131872024-02-06T22:21:17.152Z 5 100-119 120 120 120 225 225 225 F F F
131882024-02-06T22:21:17.152Z 6 120-139 112 112 112 213 213 213 F F F
131892024-02-06T22:21:17.152Z 7 140-159 120 120 120 225 225 225 F F F
131902024-02-06T22:21:17.152Z 8 160-179 119 119 119 224 224 224 F F F
131912024-02-06T22:21:17.152Z 9 180-199 120 120 120 225 225 225 F F F
131922024-02-06T22:21:17.152Z 10 200-219 118 118 118 222 222 222 F F F
131932024-02-06T22:21:17.152Z 11 220-239 118 118 118 222 222 222 F F F
131942024-02-06T22:21:17.152Z 12 240-259 120 120 120 225 225 225 F F F
131952024-02-06T22:21:17.152Z 13 260-279 118 118 118 223 223 223 F F F
131962024-02-06T22:21:17.156Z 14 280-299 120 120 120 225 225 225 F F F
131972024-02-06T22:21:17.156Z 15 300-319 120 120 120 225 225 225 F F F
131982024-02-06T22:21:17.156Z 16 320-339 120 120 120 225 225 225 F F F
131992024-02-06T22:21:17.156Z 17 340-359 120 120 120 225 225 225 F F F
132002024-02-06T22:21:17.156Z 18 360-379 120 120 120 225 225 225 F F F
132012024-02-06T22:21:17.156Z 19 380-399 118 118 118 222 222 222 F F F
132022024-02-06T22:21:17.156Z 20 400-419 118 118 118 222 222 222 F F F
132032024-02-06T22:21:17.156Z 21 420-439 119 119 119 224 224 224 F F F
132042024-02-06T22:21:17.156Z 22 440-459 119 119 119 224 224 224 F F F
132052024-02-06T22:21:17.156Z 23 460-479 120 120 120 225 225 225 F F F
132062024-02-06T22:21:17.156Z 24 480-499 113 113 113 214 214 214 F F F
132072024-02-06T22:21:17.156Z 25 500-519 120 120 120 225 225 225 F F F
132082024-02-06T22:21:17.156Z 26 520-539 119 119 119 224 224 224 F F F
132092024-02-06T22:21:17.156Z 27 540-559 116 116 116 218 218 218 F F F
132102024-02-06T22:21:17.156Z 28 560-579 119 119 119 224 224 224 F F F
132112024-02-06T22:21:17.156Z 29 580-599 118 118 118 221 221 221 F F F
132122024-02-06T22:21:17.156ZMax gen: 120, Max flush: 225
132132024-02-06T22:21:17.156Z
132142024-02-06T22:21:17.157ZBegin loop 61
132152024-02-06T22:21:17.157ZDownstairs to restart: 1
132162024-02-06T22:21:17.174Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.366511352Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626}
132172024-02-06T22:21:17.174Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.366770297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626}
132182024-02-06T22:21:17.188Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.366796904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626}
132192024-02-06T22:21:17.188Z{"msg":"Crucible 951e7963-b0e4-4d58-bf17-50a7a24bc2cd has session id: d90c8c60-6be5-41da-9fd7-9a9061c5190b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.366846657Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132202024-02-06T22:21:17.188Z{"msg":"Crucible runtime is spawned
132212024-02-06T22:21:17.188ZUpstairs opts: Upstairs UUID: 951e7963-b0e4-4d58-bf17-50a7a24bc2cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.366892521Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132222024-02-06T22:21:17.188Z{"msg":"Crucible stats registered with UUID: 951e7963-b0e4-4d58-bf17-50a7a24bc2cd","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.366918468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132232024-02-06T22:21:17.188Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.366966501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626}
132242024-02-06T22:21:17.188Z{"msg":"951e7963-b0e4-4d58-bf17-50a7a24bc2cd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367101933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132252024-02-06T22:21:17.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367137858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132262024-02-06T22:21:17.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367179082Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132272024-02-06T22:21:17.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.36719552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132282024-02-06T22:21:17.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367229966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132292024-02-06T22:21:17.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367246223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132302024-02-06T22:21:17.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367276049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132312024-02-06T22:21:17.188Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367300016Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132322024-02-06T22:21:17.189Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367339541Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132332024-02-06T22:21:17.189Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367373626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132342024-02-06T22:21:17.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367483951Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132352024-02-06T22:21:17.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367522076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132362024-02-06T22:21:17.189Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:16.367630251Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132372024-02-06T22:21:17.189Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:16.367678105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132382024-02-06T22:21:17.189Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:16.367797709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132392024-02-06T22:21:17.189Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367827195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132402024-02-06T22:21:17.189Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367842663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132412024-02-06T22:21:17.189Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.367914553Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132422024-02-06T22:21:17.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.368212473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132432024-02-06T22:21:17.189Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.368248118Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132442024-02-06T22:21:17.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.368265135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132452024-02-06T22:21:17.189Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.368289472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132462024-02-06T22:21:17.189Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.368855306Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132472024-02-06T22:21:17.189Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.368904589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132482024-02-06T22:21:17.189Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.369507928Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132492024-02-06T22:21:17.189Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.369546053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132502024-02-06T22:21:17.189Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.36956451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132512024-02-06T22:21:17.189Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.369596586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132522024-02-06T22:21:17.189Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.369628471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132532024-02-06T22:21:17.189Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:16.369662917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132542024-02-06T22:21:18.176Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.369058296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132552024-02-06T22:21:18.193Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.369213915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132562024-02-06T22:21:18.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.370005528Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132572024-02-06T22:21:18.193Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.370044763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132582024-02-06T22:21:18.193Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.370517749Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132592024-02-06T22:21:18.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371073784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132602024-02-06T22:21:18.193Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371130196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132612024-02-06T22:21:18.193Z{"msg":"[0]R flush_numbers[0..12]: [225, 225, 225, 205, 217, 225, 213, 225, 224, 225, 222, 222]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.37117563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132622024-02-06T22:21:18.193Z{"msg":"[0]R generation[0..12]: [120, 120, 120, 109, 115, 120, 112, 120, 119, 120, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371197647Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132632024-02-06T22:21:18.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371226094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132642024-02-06T22:21:18.193Z{"msg":"[1]R flush_numbers[0..12]: [225, 225, 225, 205, 217, 225, 213, 225, 224, 225, 222, 222]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371244391Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132652024-02-06T22:21:18.194Z{"msg":"[1]R generation[0..12]: [120, 120, 120, 109, 115, 120, 112, 120, 119, 120, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371269988Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"Wait for a query_work_queue command to finish before sending IO
132662024-02-06T22:21:18.194Z":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132672024-02-06T22:21:18.194Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371292555Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132682024-02-06T22:21:18.194Z{"msg":"[2]R flush_numbers[0..12]: [225, 225, 225, 205, 217, 225, 213, 225, 224, 225, 222, 222]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371311142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132692024-02-06T22:21:18.194Z{"msg":"[2]R generation[0..12]: [120, 120, 120, 109, 115, 120, 112, 120, 119, 120, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.37132943Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132702024-02-06T22:21:18.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371347527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132712024-02-06T22:21:18.194Z{"msg":"Max found gen is 121","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371381393Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132722024-02-06T22:21:18.194Z{"msg":"Generation requested: 122 >= found:121","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371408259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132732024-02-06T22:21:18.194Z{"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371437255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132742024-02-06T22:21:18.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371455563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132752024-02-06T22:21:18.195Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.37147404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132762024-02-06T22:21:18.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371490258Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132772024-02-06T22:21:18.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371507076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132782024-02-06T22:21:18.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371533532Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132792024-02-06T22:21:18.195Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.37155074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132802024-02-06T22:21:18.195Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371567268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132812024-02-06T22:21:18.195Z{"msg":"951e7963-b0e4-4d58-bf17-50a7a24bc2cd is now active with session: d90c8c60-6be5-41da-9fd7-9a9061c5190b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371583545Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132822024-02-06T22:21:18.195Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.371608082Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626,"session_id":"d90c8c60-6be5-41da-9fd7-9a9061c5190b"}
132832024-02-06T22:21:18.196Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:17.37162493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626}
132842024-02-06T22:21:18.196ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
132852024-02-06T22:21:18.196ZLoading write count information from file "/tmp/test_repair_verify.data"
132862024-02-06T22:21:18.196ZRun Repair workload
132872024-02-06T22:21:18.196Z01/30 Write block 106 len 24576 data: 8 10 13 9 10 9
132882024-02-06T22:21:18.196Z02/30 Read block 578 len 28672
132892024-02-06T22:21:18.196Z03/30 Read block 157 len 4096
132902024-02-06T22:21:18.197Z04/30 Write block 533 len 32768 data: 11 8 8 8 7 6 5 6
132912024-02-06T22:21:18.197Z05/30 Read block 491 len 20480
132922024-02-06T22:21:18.206Z06/30 Read block 174 len 40960
132932024-02-06T22:21:18.206Z07/30 Read block 501 len 20480
132942024-02-06T22:21:18.206Z08/30 Write block 349 len 40960 data: 13 14 18 19 17 16 17 15 16 14
132952024-02-06T22:21:18.206Z09/30 Write block 52 len 28672 data: 16 18 20 18 17 13 11
132962024-02-06T22:21:18.206Z10/30 Write block 302 len 28672 data: 12 12 10 12 11 9 10
132972024-02-06T22:21:18.207Z11/30 Flush
132982024-02-06T22:21:18.207Z12/30 Write block 210 len 20480 data: 11 10 9 11 13
132992024-02-06T22:21:18.207Z13/30 Read block 481 len 12288
133002024-02-06T22:21:18.207Z14/30 Read block 247 len 8192
133012024-02-06T22:21:18.207Z15/30 Write block 1 len 28672 data: 4 6 9 8 7 7 9
133022024-02-06T22:21:18.207Z16/30 Read block 434 len 16384
133032024-02-06T22:21:18.208Z17/30 Flush
133042024-02-06T22:21:18.208Z18/30 Write block 232 len 24576 data: 11 9 12 12 11 10
133052024-02-06T22:21:18.208Z19/30 Read block 535 len 24576
133062024-02-06T22:21:18.208Z20/30 Write block 363 len 20480 data: 14 12 11 9 11
133072024-02-06T22:21:18.208Z21/30 Flush
133082024-02-06T22:21:18.208Z22/30 Flush
133092024-02-06T22:21:18.208Z23/30 Write block 463 len 28672 data: 8 9 8 7 11 11 9
133102024-02-06T22:21:18.208Z24/30 Flush
133112024-02-06T22:21:18.221Z25/30 Write block 496 len 36864 data: 10 10 11 10 7 7 7 11 9
133122024-02-06T22:21:18.221Z26/30 Write block 454 len 12288 data: 9 9 7
133132024-02-06T22:21:18.221Z27/30 Read block 241 len 24576
133142024-02-06T22:21:18.221Z28/30 Write block 402 len 24576 data: 12 11 10 11 12 10
133152024-02-06T22:21:18.221Z29/30 Write block 496 len 16384 data: 11 11 12 11
133162024-02-06T22:21:18.221Z30/30 Write block 322 len 28672 data: 10 8 8 7 6 6 5
133172024-02-06T22:21:18.221Z----------------------------------------------------------------
133182024-02-06T22:21:18.221Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
133192024-02-06T22:21:18.222ZStates: Active Active Active
133202024-02-06T22:21:18.222ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133212024-02-06T22:21:18.222Z 1 Acked 1000 Write 6 Done Done Done false
133222024-02-06T22:21:18.222Z 2 Acked 1001 Read 7 Done Done Done false
133232024-02-06T22:21:18.222Z 3 Acked 1002 Read 1 Done Sent Done false
133242024-02-06T22:21:18.222Z 4 Acked 1003 Write 8 Done Sent Done false
133252024-02-06T22:21:18.222Z 5 Acked 1004 Read 5 Done Sent Done false
133262024-02-06T22:21:18.222Z 6 Acked 1005 Read 10 Done Sent Done false
133272024-02-06T22:21:18.223Z 7 Acked 1006 Read 5 Done Sent Done false
133282024-02-06T22:21:18.223Z 8 Acked 1007 Write 10 Done Sent Done false
133292024-02-06T22:21:18.223Z 9 Acked 1008 Write 7 Done Sent Done false
133302024-02-06T22:21:18.223Z 10 Acked 1009 Write 7 Done Sent Done false
133312024-02-06T22:21:18.223Z 11 Acked 1010 Flush 0 Done Sent Done false
133322024-02-06T22:21:18.223Z 12 Acked 1011 Write 5 Done Sent Done false
133332024-02-06T22:21:18.223Z 13 Acked 1012 Read 3 Done Sent Done false
133342024-02-06T22:21:18.223Z 14 Acked 1013 Read 2 Done Sent Done false
133352024-02-06T22:21:18.223Z 15 Acked 1014 Write 7 Done Sent Done false
133362024-02-06T22:21:18.223Z 16 Acked 1015 Read 4 Done Sent Done false
133372024-02-06T22:21:18.224Z 17 Acked 1016 Flush 0 Done Sent Done false
133382024-02-06T22:21:18.224Z 18 Acked 1017 Write 6 Done Sent Done false
133392024-02-06T22:21:18.224Z 19 Acked 1018 Read 6 Done Sent Done false
133402024-02-06T22:21:18.224Z 20 Acked 1019 Write 5 Done Sent Done false
133412024-02-06T22:21:18.224Z 21 Acked 1020 Flush 0 Done Sent Done false
133422024-02-06T22:21:18.224Z 22 Acked 1021 Flush 0 Done Sent Done false
133432024-02-06T22:21:18.224Z 23 Acked 1022 Write 7 Done Sent Done false
133442024-02-06T22:21:18.224Z 24 Acked 1023 Flush 0 Done Sent Done false
133452024-02-06T22:21:18.224Z 25 Acked 1024 Write 9 Done Sent Done false
133462024-02-06T22:21:18.224Z 26 Acked 1025 Write 3 Done Sent Done false
133472024-02-06T22:21:18.224Z 27 Acked 1026 Read 6 Done Sent Done false
133482024-02-06T22:21:18.225Z 28 Acked 1027 Write 6 Done Sent Done false
133492024-02-06T22:21:18.225Z 29 Acked 1028 Write 4 Done Sent Done false
133502024-02-06T22:21:18.225Z 30 Acked 1029 Write 7 Sent Sent Sent false
133512024-02-06T22:21:18.225Z STATES DS:0 DS:1 DS:2 TOTAL
133522024-02-06T22:21:18.225Z New 0 0 0 0
133532024-02-06T22:21:18.225Z Sent 1 28 1 30
133542024-02-06T22:21:18.225Z Done 29 2 29 60
133552024-02-06T22:21:18.225Z Skipped 0 0 0 0
133562024-02-06T22:21:18.225Z Error 0 0 0 0
133572024-02-06T22:21:18.225ZLast Flush: 1023 0 1023
133582024-02-06T22:21:18.226ZDownstairs last five completed:
133592024-02-06T22:21:18.226ZUpstairs last five completed: 30 29 28 27 26
133602024-02-06T22:21:18.226Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:17.404639849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1626}
133612024-02-06T22:21:18.226ZWrote out file "/tmp/test_repair_verify.data"
133622024-02-06T22:21:18.226Z
133632024-02-06T22:21:18.236ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
133642024-02-06T22:21:18.248ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133652024-02-06T22:21:18.248Z 0 000-019 122 120 122 227 225 227 F F F <---
133662024-02-06T22:21:18.248Z 1 020-039 120 120 120 225 225 225 F F F
133672024-02-06T22:21:18.248Z 2 040-059 122 120 122 226 225 226 F F F <---
133682024-02-06T22:21:18.248Z 3 060-079 109 109 109 205 205 205 F F F
133692024-02-06T22:21:18.248Z 4 080-099 115 115 115 217 217 217 F F F
133702024-02-06T22:21:18.249Z 5 100-119 122 120 122 226 225 226 F T F <---
133712024-02-06T22:21:18.249Z 6 120-139 112 112 112 213 213 213 F F F
133722024-02-06T22:21:18.249Z 7 140-159 120 120 120 225 225 225 F F F
133732024-02-06T22:21:18.249Z 8 160-179 119 119 119 224 224 224 F F F
133742024-02-06T22:21:18.249Z 9 180-199 120 120 120 225 225 225 F F F
133752024-02-06T22:21:18.249Z 10 200-219 122 118 122 227 222 227 F F F <---
133762024-02-06T22:21:18.249Z 11 220-239 122 118 122 228 222 228 F F F <---
133772024-02-06T22:21:18.249Z 12 240-259 120 120 120 225 225 225 F F F
133782024-02-06T22:21:18.249Z 13 260-279 118 118 118 223 223 223 F F F
133792024-02-06T22:21:18.249Z 14 280-299 120 120 120 225 225 225 F F F
133802024-02-06T22:21:18.249Z 15 300-319 122 120 122 226 225 226 F F F <---
133812024-02-06T22:21:18.249Z 16 320-339 120 120 120 225 225 225 T F T <---
133822024-02-06T22:21:18.250Z 17 340-359 122 120 122 226 225 226 F F F <---
133832024-02-06T22:21:18.250Z 18 360-379 122 120 122 228 225 228 F F F <---
133842024-02-06T22:21:18.250Z 19 380-399 118 118 118 222 222 222 F F F
133852024-02-06T22:21:18.250Z 20 400-419 118 118 118 222 222 222 T F T <---
133862024-02-06T22:21:18.250Z 21 420-439 119 119 119 224 224 224 F F F
133872024-02-06T22:21:18.250Z 22 440-459 119 119 119 224 224 224 T F T <---
133882024-02-06T22:21:18.250Z 23 460-479 122 120 122 230 225 230 F F F <---
133892024-02-06T22:21:18.250Z 24 480-499 113 113 113 214 214 214 T F T <---
133902024-02-06T22:21:18.250Z 25 500-519 120 120 120 225 225 225 T F T <---
133912024-02-06T22:21:18.250Z 26 520-539 122 119 122 226 224 226 F F F <---
133922024-02-06T22:21:18.250Z 27 540-559 122 116 122 226 218 226 F F F <---
133932024-02-06T22:21:18.250Z 28 560-579 119 119 119 224 224 224 F F F
133942024-02-06T22:21:18.251Z 29 580-599 118 118 118 221 221 221 F F F
133952024-02-06T22:21:18.251ZMax gen: 122, Max flush: 230
133962024-02-06T22:21:18.251ZError: Difference in extent metadata found!
133972024-02-06T22:21:18.251ZA Difference in extent metadata is expected here
133982024-02-06T22:21:18.251ZOn loop 61
133992024-02-06T22:21:18.251Z
134002024-02-06T22:21:18.251ZVerifying data now
134012024-02-06T22:21:23.025ZNo change after verify
134022024-02-06T22:21:23.025ZLoop: 61 Downstairs dump after verify (and repair):
134032024-02-06T22:21:23.441ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134042024-02-06T22:21:23.441Z 0 000-019 122 122 122 227 227 227 F F F
134052024-02-06T22:21:23.441Z 1 020-039 120 120 120 225 225 225 F F F
134062024-02-06T22:21:23.441Z 2 040-059 122 122 122 226 226 226 F F F
134072024-02-06T22:21:23.441Z 3 060-079 109 109 109 205 205 205 F F F
134082024-02-06T22:21:23.441Z 4 080-099 115 115 115 217 217 217 F F F
134092024-02-06T22:21:23.441Z 5 100-119 122 122 122 226 226 226 F F F
134102024-02-06T22:21:23.441Z 6 120-139 112 112 112 213 213 213 F F F
134112024-02-06T22:21:23.441Z 7 140-159 120 120 120 225 225 225 F F F
134122024-02-06T22:21:23.441Z 8 160-179 119 119 119 224 224 224 F F F
134132024-02-06T22:21:23.441Z 9 180-199 120 120 120 225 225 225 F F F
134142024-02-06T22:21:23.441Z 10 200-219 122 122 122 227 227 227 F F F
134152024-02-06T22:21:23.441Z 11 220-239 122 122 122 228 228 228 F F F
134162024-02-06T22:21:23.441Z 12 240-259 120 120 120 225 225 225 F F F
134172024-02-06T22:21:23.441Z 13 260-279 118 118 118 223 223 223 F F F
134182024-02-06T22:21:23.441Z 14 280-299 120 120 120 225 225 225 F F F
134192024-02-06T22:21:23.441Z 15 300-319 122 122 122 226 226 226 F F F
134202024-02-06T22:21:23.441Z 16 320-339 123 123 123 231 231 231 F F F
134212024-02-06T22:21:23.441Z 17 340-359 122 122 122 226 226 226 F F F
134222024-02-06T22:21:23.441Z 18 360-379 122 122 122 228 228 228 F F F
134232024-02-06T22:21:23.441Z 19 380-399 118 118 118 222 222 222 F F F
134242024-02-06T22:21:23.442Z 20 400-419 123 123 123 231 231 231 F F F
134252024-02-06T22:21:23.442Z 21 420-439 119 119 119 224 224 224 F F F
134262024-02-06T22:21:23.442Z 22 440-459 123 123 123 231 231 231 F F F
134272024-02-06T22:21:23.442Z 23 460-479 122 122 122 230 230 230 F F F
134282024-02-06T22:21:23.442Z 24 480-499 123 123 123 231 231 231 F F F
134292024-02-06T22:21:23.442Z 25 500-519 123 123 123 231 231 231 F F F
134302024-02-06T22:21:23.442Z 26 520-539 122 122 122 226 226 226 F F F
134312024-02-06T22:21:23.442Z 27 540-559 122 122 122 226 226 226 F F F
134322024-02-06T22:21:23.442Z 28 560-579 119 119 119 224 224 224 F F F
134332024-02-06T22:21:23.442Z 29 580-599 118 118 118 221 221 221 F F F
134342024-02-06T22:21:23.442ZMax gen: 123, Max flush: 231
134352024-02-06T22:21:23.442Z
134362024-02-06T22:21:23.442ZBegin loop 62
134372024-02-06T22:21:23.442ZDownstairs to restart: 2
134382024-02-06T22:21:23.442Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250029419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634}
134392024-02-06T22:21:23.442Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.2503141Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634}
134402024-02-06T22:21:23.442Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250335747Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634}
134412024-02-06T22:21:23.442Z{"msg":"Crucible 5dc9ff53-591d-469a-ac7a-e767751a3ddb has session id: 79885e1b-92b8-4715-8af2-d65c47c976af","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250368823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134422024-02-06T22:21:23.442Z{Crucible runtime is spawned
134432024-02-06T22:21:23.442Z"msg":"Upstairs opts: Upstairs UUID: 5dc9ff53-591d-469a-ac7a-e767751a3ddb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250415287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134442024-02-06T22:21:23.442Z{"msg":"Crucible stats registered with UUID: 5dc9ff53-591d-469a-ac7a-e767751a3ddb","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250448052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134452024-02-06T22:21:23.442Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250471389Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634}
134462024-02-06T22:21:23.442Z{"msg":"5dc9ff53-591d-469a-ac7a-e767751a3ddb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.25054021Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134472024-02-06T22:21:23.443Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250600012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134482024-02-06T22:21:23.443Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250649045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134492024-02-06T22:21:23.443Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250681091Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134502024-02-06T22:21:23.443Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250721685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134512024-02-06T22:21:23.443Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250752361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134522024-02-06T22:21:23.443Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250785397Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134532024-02-06T22:21:23.443Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250854177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134542024-02-06T22:21:23.443Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250897972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134552024-02-06T22:21:23.443Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250951374Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134562024-02-06T22:21:23.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.250998448Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134572024-02-06T22:21:23.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.251032053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134582024-02-06T22:21:23.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.251086506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134592024-02-06T22:21:23.443Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.251662418Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134602024-02-06T22:21:23.443Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.251696744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134612024-02-06T22:21:23.443Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.251711902Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134622024-02-06T22:21:23.443Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.251734508Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134632024-02-06T22:21:23.443Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.252332698Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134642024-02-06T22:21:23.443Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.252407048Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134652024-02-06T22:21:23.443Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.252728374Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134662024-02-06T22:21:23.443Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.252800794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134672024-02-06T22:21:23.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.252998288Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134682024-02-06T22:21:23.443Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.25305628Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134692024-02-06T22:21:23.443Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253081366Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134702024-02-06T22:21:23.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.25312801Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134712024-02-06T22:21:23.443Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253151957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134722024-02-06T22:21:23.444Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253178393Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134732024-02-06T22:21:23.444Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253334912Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134742024-02-06T22:21:23.444Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253791801Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134752024-02-06T22:21:23.444Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253824076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134762024-02-06T22:21:23.444Z{"msg":"[0]R flush_numbers[0..12]: [227, 225, 226, 205, 217, 226, 213, 225, 224, 225, 227, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253890557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134772024-02-06T22:21:23.444Z{"msg":"[0]R generation[0..12]: [122, 120, 122, 109, 115, 122, 112, 120, 119, 120, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253916704Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134782024-02-06T22:21:23.444Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253931982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134792024-02-06T22:21:23.444Z{"msg":"[1]R flush_numbers[0..12]: [227, 225, 226, 205, 217, 226, 213, 225, 224, 225, 227, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253946609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134802024-02-06T22:21:23.444Z{"msg":"[1]R generation[0..12]: [122, 120, 122, 109, 115, 122, 112, 120, 119, 120, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253960768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134812024-02-06T22:21:23.444Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253975066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134822024-02-06T22:21:23.444Z{"msg":"[2]R flush_numbers[0..12]: [227, 225, 226, 205, 217, 226, 213, 225, 224, 225, 227, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.253990104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134832024-02-06T22:21:23.444Z{"msg":"[2]R generation[0..12]: [122, 120, 122, 109, 115, 122, 112, 120, 119, 120, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254012061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134842024-02-06T22:21:23.444Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254026719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134852024-02-06T22:21:23.444Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254040017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO
134862024-02-06T22:21:23.444Z"79885e1b-92b8-4715-8af2-d65c47c976af"}
134872024-02-06T22:21:23.444Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254064454Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134882024-02-06T22:21:23.444Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254077772Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134892024-02-06T22:21:23.444Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.25409147Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134902024-02-06T22:21:23.444Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254112197Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134912024-02-06T22:21:23.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254124985Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134922024-02-06T22:21:23.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254138124Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134932024-02-06T22:21:23.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254157851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134942024-02-06T22:21:23.444Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254171509Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134952024-02-06T22:21:23.444Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254184168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134962024-02-06T22:21:23.444Z{"msg":"5dc9ff53-591d-469a-ac7a-e767751a3ddb is now active with session: 79885e1b-92b8-4715-8af2-d65c47c976af","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254204185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134972024-02-06T22:21:23.445Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254216513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634,"session_id":"79885e1b-92b8-4715-8af2-d65c47c976af"}
134982024-02-06T22:21:23.445Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:22.254229061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634}
134992024-02-06T22:21:23.445ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135002024-02-06T22:21:23.445ZLoading write count information from file "/tmp/test_repair_verify.data"
135012024-02-06T22:21:23.445ZRun Repair workload
135022024-02-06T22:21:23.445Z01/30 Write block 514 len 16384 data: 14 12 10 12
135032024-02-06T22:21:23.450Z02/30 Write block 321 len 40960 data: 9 11 9 9 8 7 7 6 6 5
135042024-02-06T22:21:23.450Z03/30 Write block 100 len 12288 data: 9 6 7
135052024-02-06T22:21:23.450Z04/30 Write block 53 len 24576 data: 19 21 19 18 14 12
135062024-02-06T22:21:23.450Z05/30 Write block 492 len 24576 data: 10 10 10 10 12 12
135072024-02-06T22:21:23.450Z06/30 Read block 507 len 40960
135082024-02-06T22:21:23.451Z07/30 Write block 185 len 28672 data: 17 17 15 12 11 9 10
135092024-02-06T22:21:23.451Z08/30 Flush
135102024-02-06T22:21:23.451Z09/30 Write block 305 len 40960 data: 13 12 10 11 10 11 12 12 14 14
135112024-02-06T22:21:23.451Z10/30 Write block 146 len 40960 data: 12 13 15 17 13 11 11 13 12 14
135122024-02-06T22:21:23.451Z11/30 Flush
135132024-02-06T22:21:23.451Z12/30 Write block 168 len 24576 data: 12 14 14 11 12 15
135142024-02-06T22:21:23.451Z13/30 Write block 451 len 20480 data: 8 7 7 10 10
135152024-02-06T22:21:23.451Z14/30 Write block 211 len 20480 data: 11 10 12 14 12
135162024-02-06T22:21:23.451Z15/30 Write block 41 len 4096 data: 11
135172024-02-06T22:21:23.451Z16/30 Flush
135182024-02-06T22:21:23.451Z17/30 Write block 530 len 28672 data: 12 12 10 12 9 9 9
135192024-02-06T22:21:23.451Z18/30 Read block 485 len 28672
135202024-02-06T22:21:23.451Z19/30 Write block 335 len 40960 data: 13 14 14 14 11 11 9 13 10 11
135212024-02-06T22:21:23.451Z20/30 Read block 424 len 8192
135222024-02-06T22:21:23.451Z21/30 Read block 59 len 36864
135232024-02-06T22:21:23.451Z22/30 Flush
135242024-02-06T22:21:23.451Z23/30 Write block 258 len 16384 data: 7 8 10 14
135252024-02-06T22:21:23.451Z24/30 Write block 454 len 8192 data: 11 11
135262024-02-06T22:21:23.451Z25/30 Write block 502 len 32768 data: 8 12 10 12 13 13 12 11
135272024-02-06T22:21:23.451Z26/30 Read block 86 len 16384
135282024-02-06T22:21:23.451Z27/30 Read block 0 len 8192
135292024-02-06T22:21:23.451Z28/30 Write block 437 len 20480 data: 11 10 10 9 7
135302024-02-06T22:21:23.451Z29/30 Read block 418 len 4096
135312024-02-06T22:21:23.451Z30/30 Read block 390 len 4096
135322024-02-06T22:21:23.452Z----------------------------------------------------------------
135332024-02-06T22:21:23.452Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:19
135342024-02-06T22:21:23.452ZStates: Active Active Active
135352024-02-06T22:21:23.452ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135362024-02-06T22:21:23.452Z 12 Acked 1011 Write 6 Done Done Done false
135372024-02-06T22:21:23.452Z 13 Acked 1012 Write 5 Done Done Done false
135382024-02-06T22:21:23.452Z 14 Acked 1013 Write 5 Done Done Done false
135392024-02-06T22:21:23.452Z 15 Acked 1014 Write 1 Done Done Done false
135402024-02-06T22:21:23.452Z 16 Acked 1015 Flush 0 Done Done Sent false
135412024-02-06T22:21:23.452Z 17 Acked 1016 Write 7 Done Done Sent false
135422024-02-06T22:21:23.452Z 18 Acked 1017 Read 7 Done Done Sent false
135432024-02-06T22:21:23.452Z 19 Acked 1018 Write 10 Done Done Sent false
135442024-02-06T22:21:23.452Z 20 Acked 1019 Read 2 Done Done Sent false
135452024-02-06T22:21:23.452Z 21 Acked 1020 Read 9 Done Done Sent false
135462024-02-06T22:21:23.452Z 22 Acked 1021 Flush 0 Done Done Sent false
135472024-02-06T22:21:23.452Z 23 Acked 1022 Write 4 Done Done Sent false
135482024-02-06T22:21:23.452Z 24 Acked 1023 Write 2 Done Done Sent false
135492024-02-06T22:21:23.452Z 25 Acked 1024 Write 8 Done Done Sent false
135502024-02-06T22:21:23.452Z 26 Acked 1025 Read 4 Done Done Sent false
135512024-02-06T22:21:23.452Z 27 Acked 1026 Read 2 Done Done Sent false
135522024-02-06T22:21:23.452Z 28 Acked 1027 Write 5 Done Done Sent false
135532024-02-06T22:21:23.452Z 29 Acked 1028 Read 1 Done Done Sent false
135542024-02-06T22:21:23.452Z 30 Acked 1029 Read 1 Done Done Sent false
135552024-02-06T22:21:23.452Z STATES DS:0 DS:1 DS:2 TOTAL
135562024-02-06T22:21:23.452Z New 0 0 0 0
135572024-02-06T22:21:23.453Z Sent 0 0 15 15
135582024-02-06T22:21:23.453Z Done 19 19 4 42
135592024-02-06T22:21:23.453Z Skipped 0 0 0 0
135602024-02-06T22:21:23.453Z Error 0 0 0 0
135612024-02-06T22:21:23.453ZLast Flush: 1021 1021 1010
135622024-02-06T22:21:23.453ZDownstairs last five completed: 1010 1009 1008 1007 1006
135632024-02-06T22:21:23.453ZUpstairs last five completed: 30 29 28 27 26
135642024-02-06T22:21:23.453Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:22.288387116Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1634}
135652024-02-06T22:21:23.453ZWrote out file "/tmp/test_repair_verify.data"
135662024-02-06T22:21:23.453Z
135672024-02-06T22:21:23.453ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
135682024-02-06T22:21:23.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135692024-02-06T22:21:23.453Z 0 000-019 122 122 122 227 227 227 F F F
135702024-02-06T22:21:23.453Z 1 020-039 120 120 120 225 225 225 F F F
135712024-02-06T22:21:23.453Z 2 040-059 124 124 124 234 234 232 F F T <---
135722024-02-06T22:21:23.453Z 3 060-079 109 109 109 205 205 205 F F F
135732024-02-06T22:21:23.453Z 4 080-099 115 115 115 217 217 217 F F F
135742024-02-06T22:21:23.453Z 5 100-119 124 124 124 232 232 232 F F F
135752024-02-06T22:21:23.453Z 6 120-139 112 112 112 213 213 213 F F F
135762024-02-06T22:21:23.453Z 7 140-159 124 124 124 233 233 233 F F F
135772024-02-06T22:21:23.453Z 8 160-179 124 124 119 234 234 224 F F T <---
135782024-02-06T22:21:23.454Z 9 180-199 124 124 124 232 232 232 F F F
135792024-02-06T22:21:23.454Z 10 200-219 124 124 122 234 234 227 F F T <---
135802024-02-06T22:21:23.454Z 11 220-239 122 122 122 228 228 228 F F F
135812024-02-06T22:21:23.454Z 12 240-259 120 120 120 225 225 225 T T F <---
135822024-02-06T22:21:23.454Z 13 260-279 118 118 118 223 223 223 T T F <---
135832024-02-06T22:21:23.454Z 14 280-299 120 120 120 225 225 225 F F F
135842024-02-06T22:21:23.454Z 15 300-319 124 124 124 233 233 233 F F F
135852024-02-06T22:21:23.454Z 16 320-339 124 124 124 235 235 232 F F F <---
135862024-02-06T22:21:23.454Z 17 340-359 124 124 122 235 235 226 F F F <---
135872024-02-06T22:21:23.454Z 18 360-379 122 122 122 228 228 228 F F F
135882024-02-06T22:21:23.454Z 19 380-399 118 118 118 222 222 222 F F F
135892024-02-06T22:21:23.454Z 20 400-419 123 123 123 231 231 231 F F F
135902024-02-06T22:21:23.454Z 21 420-439 119 119 119 224 224 224 T T F <---
135912024-02-06T22:21:23.454Z 22 440-459 124 124 123 234 234 231 T T T <---
135922024-02-06T22:21:23.454Z 23 460-479 122 122 122 230 230 230 F F F
135932024-02-06T22:21:23.454Z 24 480-499 124 124 124 232 232 232 F F F
135942024-02-06T22:21:23.454Z 25 500-519 124 124 124 232 232 232 T T F <---
135952024-02-06T22:21:23.454Z 26 520-539 124 124 122 235 235 226 F F F <---
135962024-02-06T22:21:23.454Z 27 540-559 122 122 122 226 226 226 F F F
135972024-02-06T22:21:23.454Z 28 560-579 119 119 119 224 224 224 F F F
135982024-02-06T22:21:23.454Z 29 580-599 118 118 118 221 221 221 F F F
135992024-02-06T22:21:23.454ZMax gen: 124, Max flush: 235
136002024-02-06T22:21:23.454ZError: Difference in extent metadata found!
136012024-02-06T22:21:23.454ZA Difference in extent metadata is expected here
136022024-02-06T22:21:23.455ZOn loop 62
136032024-02-06T22:21:23.460Z
136042024-02-06T22:21:23.460ZVerifying data now
136052024-02-06T22:21:28.681ZNo change after verify
136062024-02-06T22:21:28.681ZLoop: 62 Downstairs dump after verify (and repair):
136072024-02-06T22:21:28.696ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136082024-02-06T22:21:28.696Z 0 000-019 122 122 122 227 227 227 F F F
136092024-02-06T22:21:28.696Z 1 020-039 120 120 120 225 225 225 F F F
136102024-02-06T22:21:28.696Z 2 040-059 124 124 124 234 234 234 F F F
136112024-02-06T22:21:28.696Z 3 060-079 109 109 109 205 205 205 F F F
136122024-02-06T22:21:28.696Z 4 080-099 115 115 115 217 217 217 F F F
136132024-02-06T22:21:28.696Z 5 100-119 124 124 124 232 232 232 F F F
136142024-02-06T22:21:28.696Z 6 120-139 112 112 112 213 213 213 F F F
136152024-02-06T22:21:28.696Z 7 140-159 124 124 124 233 233 233 F F F
136162024-02-06T22:21:28.699Z 8 160-179 124 124 124 234 234 234 F F F
136172024-02-06T22:21:28.700Z 9 180-199 124 124 124 232 232 232 F F F
136182024-02-06T22:21:28.700Z 10 200-219 124 124 124 234 234 234 F F F
136192024-02-06T22:21:28.700Z 11 220-239 122 122 122 228 228 228 F F F
136202024-02-06T22:21:28.700Z 12 240-259 125 125 125 236 236 236 F F F
136212024-02-06T22:21:28.700Z 13 260-279 125 125 125 236 236 236 F F F
136222024-02-06T22:21:28.700Z 14 280-299 120 120 120 225 225 225 F F F
136232024-02-06T22:21:28.700Z 15 300-319 124 124 124 233 233 233 F F F
136242024-02-06T22:21:28.700Z 16 320-339 124 124 124 235 235 235 F F F
136252024-02-06T22:21:28.700Z 17 340-359 124 124 124 235 235 235 F F F
136262024-02-06T22:21:28.700Z 18 360-379 122 122 122 228 228 228 F F F
136272024-02-06T22:21:28.700Z 19 380-399 118 118 118 222 222 222 F F F
136282024-02-06T22:21:28.700Z 20 400-419 123 123 123 231 231 231 F F F
136292024-02-06T22:21:28.700Z 21 420-439 125 125 125 236 236 236 F F F
136302024-02-06T22:21:28.700Z 22 440-459 125 125 125 236 236 236 F F F
136312024-02-06T22:21:28.700Z 23 460-479 122 122 122 230 230 230 F F F
136322024-02-06T22:21:28.700Z 24 480-499 124 124 124 232 232 232 F F F
136332024-02-06T22:21:28.700Z 25 500-519 125 125 125 236 236 236 F F F
136342024-02-06T22:21:28.700Z 26 520-539 124 124 124 235 235 235 F F F
136352024-02-06T22:21:28.700Z 27 540-559 122 122 122 226 226 226 F F F
136362024-02-06T22:21:28.700Z 28 560-579 119 119 119 224 224 224 F F F
136372024-02-06T22:21:28.700Z 29 580-599 118 118 118 221 221 221 F F F
136382024-02-06T22:21:28.700ZMax gen: 125, Max flush: 236
136392024-02-06T22:21:28.700Z
136402024-02-06T22:21:28.700ZBegin loop 63
136412024-02-06T22:21:28.701ZDownstairs to restart: 0
136422024-02-06T22:21:28.718Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912412421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642}
136432024-02-06T22:21:28.718Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912687234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642}
136442024-02-06T22:21:28.718ZCrucible runtime is spawned
136452024-02-06T22:21:28.723Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912730578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642}
136462024-02-06T22:21:28.723Z{"msg":"Crucible ffc6ff32-97a9-48a3-8588-22d452da285d has session id: 09f1a17c-705c-480a-ace6-57cdd60cdd37","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912775172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136472024-02-06T22:21:28.723Z{"msg":"Upstairs opts: Upstairs UUID: ffc6ff32-97a9-48a3-8588-22d452da285d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912808427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136482024-02-06T22:21:28.723Z{"msg":"Crucible stats registered with UUID: ffc6ff32-97a9-48a3-8588-22d452da285d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912829624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136492024-02-06T22:21:28.723Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912853371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642}
136502024-02-06T22:21:28.723Z{"msg":"ffc6ff32-97a9-48a3-8588-22d452da285d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912880258Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136512024-02-06T22:21:28.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912901355Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136522024-02-06T22:21:28.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912916443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136532024-02-06T22:21:28.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912930351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136542024-02-06T22:21:28.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912956967Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136552024-02-06T22:21:28.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.912971016Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136562024-02-06T22:21:28.723Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91300873Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136572024-02-06T22:21:28.723Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.913053744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136582024-02-06T22:21:28.723Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91308636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136592024-02-06T22:21:28.723Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.913113086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136602024-02-06T22:21:28.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.913272145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136612024-02-06T22:21:28.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.913337636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136622024-02-06T22:21:28.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.913385589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136632024-02-06T22:21:28.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.913969481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136642024-02-06T22:21:28.724Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.914037461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136652024-02-06T22:21:28.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.914072077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136662024-02-06T22:21:28.724Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.914108502Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136672024-02-06T22:21:28.724Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.914589497Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136682024-02-06T22:21:28.724Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.914771922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136692024-02-06T22:21:28.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91508483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136702024-02-06T22:21:28.724Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915150551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136712024-02-06T22:21:28.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915181057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136722024-02-06T22:21:28.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915338305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136732024-02-06T22:21:28.724Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915373821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136742024-02-06T22:21:28.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915389619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136752024-02-06T22:21:28.724Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915403297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136762024-02-06T22:21:28.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915423914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136772024-02-06T22:21:28.729Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.915884002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136782024-02-06T22:21:28.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91633937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136792024-02-06T22:21:28.729Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916377325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136802024-02-06T22:21:28.729Z{"msg":"[0]R flush_numbers[0..12]: [227, 225, 234, 205, 217, 232, 213, 233, 234, 232, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916394643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136812024-02-06T22:21:28.729Z{"msg":"[0]R generation[0..12]: [122, 120, 124, 109, 115, 124, 112, 124, 124, 124, 124, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91641668Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136822024-02-06T22:21:28.729Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916431428Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136832024-02-06T22:21:28.729Z{"msg":"[1]R flush_numbers[0..12]: [227, 225, 234, 205, 217, 232, 213, 233, 234, 232, 234, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916452975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136842024-02-06T22:21:28.729Z{"msg":"[1]R generation[0..12]: [122, 120, 124, 109, 115, 124, 112, 124, 124, 124, 124, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916467653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136852024-02-06T22:21:28.729Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91648849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136862024-02-06T22:21:28.729Z{"msg":"[2]R flush_numbers[0..12]: [227, 225, 234, 205, 217, 232, 213, 233, 234, 232, 234, 228]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
136872024-02-06T22:21:28.729Z,"time":"2024-02-06T22:21:27.916503258Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136882024-02-06T22:21:28.729Z{"msg":"[2]R generation[0..12]: [122, 120, 124, 109, 115, 124, 112, 124, 124, 124, 124, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916530324Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136892024-02-06T22:21:28.729Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916544893Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136902024-02-06T22:21:28.729Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91656559Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136912024-02-06T22:21:28.729Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916579308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136922024-02-06T22:21:28.729Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916603445Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136932024-02-06T22:21:28.729Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916623282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136942024-02-06T22:21:28.730Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916643029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136952024-02-06T22:21:28.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916668066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136962024-02-06T22:21:28.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916683744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136972024-02-06T22:21:28.730Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916697912Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136982024-02-06T22:21:28.730Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916757244Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
136992024-02-06T22:21:28.730Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.91678293Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
137002024-02-06T22:21:28.730Z{"msg":"ffc6ff32-97a9-48a3-8588-22d452da285d is now active with session: 09f1a17c-705c-480a-ace6-57cdd60cdd37","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916814086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
137012024-02-06T22:21:28.730Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916853981Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642,"session_id":"09f1a17c-705c-480a-ace6-57cdd60cdd37"}
137022024-02-06T22:21:28.730Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:27.916877688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642}
137032024-02-06T22:21:28.730ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137042024-02-06T22:21:28.730ZLoading write count information from file "/tmp/test_repair_verify.data"
137052024-02-06T22:21:28.730ZRun Repair workload
137062024-02-06T22:21:28.730Z01/30 Write block 436 len 20480 data: 10 12 11 11 10
137072024-02-06T22:21:28.730Z02/30 Flush
137082024-02-06T22:21:28.730Z03/30 Flush
137092024-02-06T22:21:28.730Z04/30 Write block 253 len 36864 data: 7 9 8 5 7 8 9 11 15
137102024-02-06T22:21:28.733Z05/30 Write block 290 len 16384 data: 14 14 11 12
137112024-02-06T22:21:28.733Z06/30 Write block 202 len 24576 data: 13 12 13 11 11 9
137122024-02-06T22:21:28.733Z07/30 Write block 160 len 36864 data: 10 9 11 9 9 7 8 10 13
137132024-02-06T22:21:28.733Z08/30 Write block 551 len 20480 data: 8 9 9 8 7
137142024-02-06T22:21:28.733Z09/30 Read block 390 len 36864
137152024-02-06T22:21:28.733Z10/30 Flush
137162024-02-06T22:21:28.736Z11/30 Write block 162 len 16384 data: 12 10 10 8
137172024-02-06T22:21:28.736Z12/30 Write block 565 len 28672 data: 11 11 13 14 12 11 15
137182024-02-06T22:21:28.739Z13/30 Write block 300 len 4096 data: 12
137192024-02-06T22:21:28.739Z14/30 Read block 554 len 28672
137202024-02-06T22:21:28.739Z15/30 Read block 406 len 24576
137212024-02-06T22:21:28.742Z16/30 Write block 485 len 16384 data: 8 9 9 8
137222024-02-06T22:21:28.742Z17/30 Flush
137232024-02-06T22:21:28.742Z18/30 Flush
137242024-02-06T22:21:28.748Z19/30 Write block 423 len 24576 data: 9 10 10 12 10 11
137252024-02-06T22:21:28.748Z20/30 Read block 264 len 8192
137262024-02-06T22:21:28.748Z21/30 Read block 337 len 28672
137272024-02-06T22:21:28.748Z22/30 Read block 524 len 16384
137282024-02-06T22:21:28.751Z23/30 Read block 159 len 24576
137292024-02-06T22:21:28.751Z24/30 Read block 507 len 4096
137302024-02-06T22:21:28.751Z25/30 Read block 464 len 24576
137312024-02-06T22:21:28.751Z26/30 Write block 584 len 28672 data: 9 12 12 13 13 16 19
137322024-02-06T22:21:28.751Z27/30 Read block 294 len 8192
137332024-02-06T22:21:28.751Z28/30 Write block 533 len 24576 data: 13 10 10 10 8 7
137342024-02-06T22:21:28.751Z29/30 Read block 422 len 4096
137352024-02-06T22:21:28.755Z30/30 Read block 105 len 32768
137362024-02-06T22:21:28.755Z----------------------------------------------------------------
137372024-02-06T22:21:28.755Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
137382024-02-06T22:21:28.755ZStates: Active Active Active
137392024-02-06T22:21:28.755ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
137402024-02-06T22:21:28.755Z 1 Acked 1000 Write 5 Sent Done Done false
137412024-02-06T22:21:28.755Z 2 Acked 1001 Flush 0 Sent Done Done false
137422024-02-06T22:21:28.755Z 3 Acked 1002 Flush 0 Sent Done Done false
137432024-02-06T22:21:28.755Z 4 Acked 1003 Write 9 Sent Done Done false
137442024-02-06T22:21:28.755Z 5 Acked 1004 Write 4 Sent Done Done false
137452024-02-06T22:21:28.755Z 6 Acked 1005 Write 6 Sent Done Done false
137462024-02-06T22:21:28.755Z 7 Acked 1006 Write 9 Sent Done Done false
137472024-02-06T22:21:28.755Z 8 Acked 1007 Write 5 Sent Done Done false
137482024-02-06T22:21:28.755Z 9 Acked 1008 Read 9 Sent Done Done false
137492024-02-06T22:21:28.755Z 10 Acked 1009 Flush 0 Sent Done Done false
137502024-02-06T22:21:28.756Z 11 Acked 1010 Write 4 Sent Done Done false
137512024-02-06T22:21:28.756Z 12 Acked 1011 Write 7 Sent Done Done false
137522024-02-06T22:21:28.756Z 13 Acked 1012 Write 1 Sent Done Done false
137532024-02-06T22:21:28.756Z 14 Acked 1013 Read 7 Sent Done Done false
137542024-02-06T22:21:28.756Z 15 Acked 1014 Read 6 Sent Done Done false
137552024-02-06T22:21:28.756Z 16 Acked 1015 Write 4 Sent Done Done false
137562024-02-06T22:21:28.756Z 17 Acked 1016 Flush 0 Sent Done Done false
137572024-02-06T22:21:28.756Z 18 Acked 1017 Flush 0 Sent Done Done false
137582024-02-06T22:21:28.756Z 19 Acked 1018 Write 6 Sent Done Done false
137592024-02-06T22:21:28.756Z 20 Acked 1019 Read 2 Sent Done Done false
137602024-02-06T22:21:28.756Z 21 Acked 1020 Read 7 Sent Done Done false
137612024-02-06T22:21:28.756Z 22 Acked 1021 Read 4 Sent Done Done false
137622024-02-06T22:21:28.756Z 23 Acked 1022 Read 6 Sent Done Done false
137632024-02-06T22:21:28.756Z 24 Acked 1023 Read 1 Sent Done Done false
137642024-02-06T22:21:28.756Z 25 Acked 1024 Read 6 Sent Done Done false
137652024-02-06T22:21:28.756Z 26 Acked 1025 Write 7 Sent Done Done false
137662024-02-06T22:21:28.756Z 27 Acked 1026 Read 2 Sent Done Done false
137672024-02-06T22:21:28.756Z 28 Acked 1027 Write 6 Sent Done Done false
137682024-02-06T22:21:28.756Z 29 Acked 1028 Read 1 Sent Done Done false
137692024-02-06T22:21:28.756Z 30 Acked 1029 Read 8 Sent Sent Done false
137702024-02-06T22:21:28.756Z STATES DS:0 DS:1 DS:2 TOTAL
137712024-02-06T22:21:28.756Z New 0 0 0 0
137722024-02-06T22:21:28.756Z Sent 30 1 0 31
137732024-02-06T22:21:28.756Z Done 0 29 30 59
137742024-02-06T22:21:28.756Z Skipped 0 0 0 0
137752024-02-06T22:21:28.757Z Error 0 0 0 0
137762024-02-06T22:21:28.757ZLast Flush: 0 1017 1017
137772024-02-06T22:21:28.757ZDownstairs last five completed:
137782024-02-06T22:21:28.757ZUpstairs last five completed: 30 29 28 27 26
137792024-02-06T22:21:28.757Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:27.947299487Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1642}
137802024-02-06T22:21:28.757ZWrote out file "/tmp/test_repair_verify.data"
137812024-02-06T22:21:28.760Z
137822024-02-06T22:21:28.760ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
137832024-02-06T22:21:28.775ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137842024-02-06T22:21:28.775Z 0 000-019 122 122 122 227 227 227 F F F
137852024-02-06T22:21:28.776Z 1 020-039 120 120 120 225 225 225 F F F
137862024-02-06T22:21:28.776Z 2 040-059 124 124 124 234 234 234 F F F
137872024-02-06T22:21:28.776Z 3 060-079 109 109 109 205 205 205 F F F
137882024-02-06T22:21:28.776Z 4 080-099 115 115 115 217 217 217 F F F
137892024-02-06T22:21:28.776Z 5 100-119 124 124 124 232 232 232 F F F
137902024-02-06T22:21:28.776Z 6 120-139 112 112 112 213 213 213 F F F
137912024-02-06T22:21:28.776Z 7 140-159 124 124 124 233 233 233 F F F
137922024-02-06T22:21:28.776Z 8 160-179 124 126 126 234 240 240 F F F <---
137932024-02-06T22:21:28.776Z 9 180-199 124 124 124 232 232 232 F F F
137942024-02-06T22:21:28.776Z 10 200-219 124 126 126 234 239 239 F F F <---
137952024-02-06T22:21:28.776Z 11 220-239 122 122 122 228 228 228 F F F
137962024-02-06T22:21:28.779Z 12 240-259 125 126 126 236 239 239 F F F <---
137972024-02-06T22:21:28.779Z 13 260-279 125 126 126 236 239 239 F F F <---
137982024-02-06T22:21:28.779Z 14 280-299 120 126 126 225 239 239 F F F <---
137992024-02-06T22:21:28.779Z 15 300-319 124 126 126 233 240 240 F F F <---
138002024-02-06T22:21:28.779Z 16 320-339 124 124 124 235 235 235 F F F
138012024-02-06T22:21:28.779Z 17 340-359 124 124 124 235 235 235 F F F
138022024-02-06T22:21:28.779Z 18 360-379 122 122 122 228 228 228 F F F
138032024-02-06T22:21:28.779Z 19 380-399 118 118 118 222 222 222 F F F
138042024-02-06T22:21:28.779Z 20 400-419 123 123 123 231 231 231 F F F
138052024-02-06T22:21:28.779Z 21 420-439 125 126 126 236 237 237 F T T <---
138062024-02-06T22:21:28.779Z 22 440-459 125 126 126 236 237 237 F F F <---
138072024-02-06T22:21:28.779Z 23 460-479 122 122 122 230 230 230 F F F
138082024-02-06T22:21:28.779Z 24 480-499 124 126 126 232 240 240 F F F <---
138092024-02-06T22:21:28.780Z 25 500-519 125 125 125 236 236 236 F F F
138102024-02-06T22:21:28.780Z 26 520-539 124 124 124 235 235 235 F T T <---
138112024-02-06T22:21:28.780Z 27 540-559 122 126 126 226 239 239 F F F <---
138122024-02-06T22:21:28.780Z 28 560-579 119 126 126 224 240 240 F F F <---
138132024-02-06T22:21:28.780Z 29 580-599 118 118 118 221 221 221 F T T <---
138142024-02-06T22:21:28.780ZMax gen: 126, Max flush: 240
138152024-02-06T22:21:28.780ZError: Difference in extent metadata found!
138162024-02-06T22:21:28.780ZA Difference in extent metadata is expected here
138172024-02-06T22:21:28.780ZOn loop 63
138182024-02-06T22:21:28.780Z
138192024-02-06T22:21:28.783ZVerifying data now
138202024-02-06T22:21:33.408ZNo change after verify
138212024-02-06T22:21:33.409ZLoop: 63 Downstairs dump after verify (and repair):
138222024-02-06T22:21:33.425ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138232024-02-06T22:21:33.425Z 0 000-019 122 122 122 227 227 227 F F F
138242024-02-06T22:21:33.425Z 1 020-039 120 120 120 225 225 225 F F F
138252024-02-06T22:21:33.425Z 2 040-059 124 124 124 234 234 234 F F F
138262024-02-06T22:21:33.425Z 3 060-079 109 109 109 205 205 205 F F F
138272024-02-06T22:21:33.425Z 4 080-099 115 115 115 217 217 217 F F F
138282024-02-06T22:21:33.425Z 5 100-119 124 124 124 232 232 232 F F F
138292024-02-06T22:21:33.425Z 6 120-139 112 112 112 213 213 213 F F F
138302024-02-06T22:21:33.425Z 7 140-159 124 124 124 233 233 233 F F F
138312024-02-06T22:21:33.426Z 8 160-179 126 126 126 240 240 240 F F F
138322024-02-06T22:21:33.426Z 9 180-199 124 124 124 232 232 232 F F F
138332024-02-06T22:21:33.426Z 10 200-219 126 126 126 239 239 239 F F F
138342024-02-06T22:21:33.426Z 11 220-239 122 122 122 228 228 228 F F F
138352024-02-06T22:21:33.426Z 12 240-259 126 126 126 239 239 239 F F F
138362024-02-06T22:21:33.426Z 13 260-279 126 126 126 239 239 239 F F F
138372024-02-06T22:21:33.429Z 14 280-299 126 126 126 239 239 239 F F F
138382024-02-06T22:21:33.429Z 15 300-319 126 126 126 240 240 240 F F F
138392024-02-06T22:21:33.429Z 16 320-339 124 124 124 235 235 235 F F F
138402024-02-06T22:21:33.429Z 17 340-359 124 124 124 235 235 235 F F F
138412024-02-06T22:21:33.429Z 18 360-379 122 122 122 228 228 228 F F F
138422024-02-06T22:21:33.429Z 19 380-399 118 118 118 222 222 222 F F F
138432024-02-06T22:21:33.430Z 20 400-419 123 123 123 231 231 231 F F F
138442024-02-06T22:21:33.430Z 21 420-439 127 127 127 241 241 241 F F F
138452024-02-06T22:21:33.430Z 22 440-459 126 126 126 237 237 237 F F F
138462024-02-06T22:21:33.430Z 23 460-479 122 122 122 230 230 230 F F F
138472024-02-06T22:21:33.430Z 24 480-499 126 126 126 240 240 240 F F F
138482024-02-06T22:21:33.430Z 25 500-519 125 125 125 236 236 236 F F F
138492024-02-06T22:21:33.430Z 26 520-539 127 127 127 241 241 241 F F F
138502024-02-06T22:21:33.430Z 27 540-559 126 126 126 239 239 239 F F F
138512024-02-06T22:21:33.430Z 28 560-579 126 126 126 240 240 240 F F F
138522024-02-06T22:21:33.430Z 29 580-599 127 127 127 241 241 241 F F F
138532024-02-06T22:21:33.430ZMax gen: 127, Max flush: 241
138542024-02-06T22:21:33.430Z
138552024-02-06T22:21:33.430ZBegin loop 64
138562024-02-06T22:21:33.430ZDownstairs to restart: 2
138572024-02-06T22:21:33.445Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.638941918Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650}
138582024-02-06T22:21:33.445Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\nCrucible runtime is spawned
138592024-02-06T22:21:33.451Z git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639241587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650}
138602024-02-06T22:21:33.451Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639291111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650}
138612024-02-06T22:21:33.451Z{"msg":"Crucible c78b4194-ebd6-44de-b597-256e321769ed has session id: 4fbabdcd-0575-44dd-8ba1-53fe805685ec","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639318477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138622024-02-06T22:21:33.451Z{"msg":"Upstairs opts: Upstairs UUID: c78b4194-ebd6-44de-b597-256e321769ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639364941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138632024-02-06T22:21:33.451Z{"msg":"Crucible stats registered with UUID: c78b4194-ebd6-44de-b597-256e321769ed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639383878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138642024-02-06T22:21:33.451Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639414514Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650}
138652024-02-06T22:21:33.451Z{"msg":"c78b4194-ebd6-44de-b597-256e321769ed active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639448279Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138662024-02-06T22:21:33.451Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639471466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138672024-02-06T22:21:33.451Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639486934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138682024-02-06T22:21:33.451Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639508391Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138692024-02-06T22:21:33.451Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639523249Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138702024-02-06T22:21:33.451Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639556135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138712024-02-06T22:21:33.451Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.63959473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138722024-02-06T22:21:33.451Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639638444Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138732024-02-06T22:21:33.451Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639670599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138742024-02-06T22:21:33.451Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639698645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138752024-02-06T22:21:33.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639821209Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138762024-02-06T22:21:33.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639873662Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138772024-02-06T22:21:33.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.639892339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138782024-02-06T22:21:33.452Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.640689192Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138792024-02-06T22:21:33.452Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.640744714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138802024-02-06T22:21:33.452Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.64077451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138812024-02-06T22:21:33.452Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.640809865Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138822024-02-06T22:21:33.452Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.641302509Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138832024-02-06T22:21:33.452Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.641477275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138842024-02-06T22:21:33.452Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.641510921Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138852024-02-06T22:21:33.452Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.641533357Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138862024-02-06T22:21:33.452Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.641824658Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138872024-02-06T22:21:33.452Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.641859204Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138882024-02-06T22:21:33.452Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.641876481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138892024-02-06T22:21:33.452Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642054757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138902024-02-06T22:21:33.452Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642090822Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138912024-02-06T22:21:33.452Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.64210687Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138922024-02-06T22:21:33.452Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642120958Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138932024-02-06T22:21:33.458Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642607032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138942024-02-06T22:21:33.458Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642657686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138952024-02-06T22:21:33.458Z{"msg":"[0]R flush_numbers[0..12]: [227, 225, 234, 205, 217, 232, 213, 233, 240, 232, 239, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642678413Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138962024-02-06T22:21:33.459Z{"msg":"[0]R generation[0..12]: [122, 120, 124, 109, 115, 124, 112, 124, 126, 124, 126, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.64270409Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138972024-02-06T22:21:33.459Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642720557Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138982024-02-06T22:21:33.459Z{"msg":"[1]R flush_numbers[0..12]: [227, 225, 234, 205, 217, 232, 213, 233, 240, 232, 239, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642736065Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
138992024-02-06T22:21:33.459Z{"msg":"[1]R generation[0..12]: [122, 120, 124, 109, 115, 124, 112, 124, 126, 124, 126, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642751013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139002024-02-06T22:21:33.459Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642766431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139012024-02-06T22:21:33.459Z{"msg":"[2]R flush_numbers[0..12]: [227, 225, 234, 205, 217, 232, 213, 233, 240, 232, 239, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642788548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139022024-02-06T22:21:33.459Z{"msg":"[2]R generation[0..12]: [122, 120, 124, 109, 115, 124, 112, 124, 126, 124, 126, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642802916Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139032024-02-06T22:21:33.459Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
139042024-02-06T22:21:33.459Z2024-02-06T22:21:32.642824403Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139052024-02-06T22:21:33.459Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642840951Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139062024-02-06T22:21:33.459Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642854979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139072024-02-06T22:21:33.459Z{"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642875326Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139082024-02-06T22:21:33.459Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642889464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139092024-02-06T22:21:33.460Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642903472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139102024-02-06T22:21:33.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642916691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139112024-02-06T22:21:33.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642930439Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139122024-02-06T22:21:33.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642943717Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139132024-02-06T22:21:33.460Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642957065Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139142024-02-06T22:21:33.460Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642977342Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139152024-02-06T22:21:33.460Z{"msg":"c78b4194-ebd6-44de-b597-256e321769ed is now active with session: 4fbabdcd-0575-44dd-8ba1-53fe805685ec","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.642991081Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139162024-02-06T22:21:33.460Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.643003099Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650,"session_id":"4fbabdcd-0575-44dd-8ba1-53fe805685ec"}
139172024-02-06T22:21:33.460Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:32.643015307Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650}
139182024-02-06T22:21:33.460ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
139192024-02-06T22:21:33.460ZLoading write count information from file "/tmp/test_repair_verify.data"
139202024-02-06T22:21:33.460ZRun Repair workload
139212024-02-06T22:21:33.460Z01/30 Write block 106 len 36864 data: 9 11 14 10 11 10 10 11 13
139222024-02-06T22:21:33.460Z02/30 Read block 0 len 36864
139232024-02-06T22:21:33.461Z03/30 Write block 183 len 12288 data: 16 18 18
139242024-02-06T22:21:33.461Z04/30 Write block 341 len 16384 data: 10 14 11 12
139252024-02-06T22:21:33.461Z05/30 Write block 183 len 32768 data: 17 19 19 18 16 13 12 10
139262024-02-06T22:21:33.461Z06/30 Read block 224 len 16384
139272024-02-06T22:21:33.475Z07/30 Read block 137 len 32768
139282024-02-06T22:21:33.475Z08/30 Flush
139292024-02-06T22:21:33.475Z09/30 Write block 223 len 16384 data: 13 13 10 10
139302024-02-06T22:21:33.475Z10/30 Read block 349 len 16384
139312024-02-06T22:21:33.475Z11/30 Read block 465 len 24576
139322024-02-06T22:21:33.475Z12/30 Write block 38 len 16384 data: 7 7 13 12
139332024-02-06T22:21:33.475Z13/30 Write block 531 len 24576 data: 13 11 14 11 11 11
139342024-02-06T22:21:33.475Z14/30 Write block 175 len 12288 data: 14 12 11
139352024-02-06T22:21:33.475Z15/30 Write block 301 len 32768 data: 13 13 13 11 14 13 11 12
139362024-02-06T22:21:33.479Z16/30 Read block 346 len 20480
139372024-02-06T22:21:33.479Z17/30 Flush
139382024-02-06T22:21:33.479Z18/30 Write block 105 len 8192 data: 7 10
139392024-02-06T22:21:33.479Z19/30 Write block 132 len 24576 data: 11 12 11 9 8 8
139402024-02-06T22:21:33.479Z20/30 Read block 120 len 32768
139412024-02-06T22:21:33.479Z21/30 Write block 152 len 32768 data: 12 14 13 15 12 14 10 11
139422024-02-06T22:21:33.479Z22/30 Read block 18 len 24576
139432024-02-06T22:21:33.479Z23/30 Read block 152 len 16384
139442024-02-06T22:21:33.479Z24/30 Read block 295 len 16384
139452024-02-06T22:21:33.479Z25/30 Write block 275 len 36864 data: 9 11 11 12 12 12 11 9 6
139462024-02-06T22:21:33.479Z26/30 Read block 205 len 28672
139472024-02-06T22:21:33.479Z27/30 Read block 196 len 32768
139482024-02-06T22:21:33.479Z28/30 Read block 376 len 40960
139492024-02-06T22:21:33.485Z29/30 Read block 533 len 8192
139502024-02-06T22:21:33.485Z30/30 Write block 315 len 16384 data: 15 14 13 12
139512024-02-06T22:21:33.485Z----------------------------------------------------------------
139522024-02-06T22:21:33.485Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:22
139532024-02-06T22:21:33.485ZStates: Active Active Active
139542024-02-06T22:21:33.485ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
139552024-02-06T22:21:33.485Z 9 Acked 1008 Write 4 Done Done Done false
139562024-02-06T22:21:33.485Z 10 Acked 1009 Read 4 Done Done Done false
139572024-02-06T22:21:33.485Z 11 Acked 1010 Read 6 Done Done Done false
139582024-02-06T22:21:33.485Z 12 Acked 1011 Write 4 Done Done Done false
139592024-02-06T22:21:33.485Z 13 Acked 1012 Write 6 Done Done Done false
139602024-02-06T22:21:33.485Z 14 Acked 1013 Write 3 Done Done Sent false
139612024-02-06T22:21:33.485Z 15 Acked 1014 Write 8 Done Done Sent false
139622024-02-06T22:21:33.485Z 16 Acked 1015 Read 5 Done Done Sent false
139632024-02-06T22:21:33.485Z 17 Acked 1016 Flush 0 Done Done Sent false
139642024-02-06T22:21:33.485Z 18 Acked 1017 Write 2 Done Done Sent false
139652024-02-06T22:21:33.485Z 19 Acked 1018 Write 6 Done Done Sent false
139662024-02-06T22:21:33.485Z 20 Acked 1019 Read 8 Done Done Sent false
139672024-02-06T22:21:33.485Z 21 Acked 1020 Write 8 Done Done Sent false
139682024-02-06T22:21:33.485Z 22 Acked 1021 Read 6 Done Done Sent false
139692024-02-06T22:21:33.485Z 23 Acked 1022 Read 4 Done Done Sent false
139702024-02-06T22:21:33.486Z 24 Acked 1023 Read 4 Done Done Sent false
139712024-02-06T22:21:33.486Z 25 Acked 1024 Write 9 Done Done Sent false
139722024-02-06T22:21:33.486Z 26 Acked 1025 Read 7 Done Done Sent false
139732024-02-06T22:21:33.486Z 27 Acked 1026 Read 8 Done Done Sent false
139742024-02-06T22:21:33.486Z 28 Acked 1027 Read 10 Done Done Sent false
139752024-02-06T22:21:33.486Z 29 Acked 1028 Read 2 Done Done Sent false
139762024-02-06T22:21:33.486Z 30 Acked 1029 Write 4 Sent Sent Sent false
139772024-02-06T22:21:33.486Z STATES DS:0 DS:1 DS:2 TOTAL
139782024-02-06T22:21:33.486Z New 0 0 0 0
139792024-02-06T22:21:33.486Z Sent 1 1 17 19
139802024-02-06T22:21:33.486Z Done 21 21 5 47
139812024-02-06T22:21:33.486Z Skipped 0 0 0 0
139822024-02-06T22:21:33.486Z Error 0 0 0 0
139832024-02-06T22:21:33.486ZLast Flush: 1016 1016 1007
139842024-02-06T22:21:33.486ZDownstairs last five completed: 1007 1006 1005 1004 1003
139852024-02-06T22:21:33.486ZUpstairs last five completed: 30 29 28 27 26
139862024-02-06T22:21:33.486Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:32.67452542Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1650}
139872024-02-06T22:21:33.486ZWrote out file "/tmp/test_repair_verify.data"
139882024-02-06T22:21:33.491Z
139892024-02-06T22:21:33.491ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
139902024-02-06T22:21:33.508ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139912024-02-06T22:21:33.508Z 0 000-019 122 122 122 227 227 227 F F F
139922024-02-06T22:21:33.508Z 1 020-039 128 128 120 243 243 225 F F T <---
139932024-02-06T22:21:33.508Z 2 040-059 128 128 124 243 243 234 F F T <---
139942024-02-06T22:21:33.508Z 3 060-079 109 109 109 205 205 205 F F F
139952024-02-06T22:21:33.508Z 4 080-099 115 115 115 217 217 217 F F F
139962024-02-06T22:21:33.508Z 5 100-119 128 128 128 242 242 242 T T F <---
139972024-02-06T22:21:33.508Z 6 120-139 112 112 112 213 213 213 T T F <---
139982024-02-06T22:21:33.508Z 7 140-159 124 124 124 233 233 233 T T F <---
139992024-02-06T22:21:33.508Z 8 160-179 128 128 126 243 243 240 F F F <---
140002024-02-06T22:21:33.508Z 9 180-199 128 128 128 242 242 242 F F F
140012024-02-06T22:21:33.513Z 10 200-219 126 126 126 239 239 239 F F F
140022024-02-06T22:21:33.513Z 11 220-239 128 128 122 243 243 228 F F T <---
140032024-02-06T22:21:33.513Z 12 240-259 126 126 126 239 239 239 F F F
140042024-02-06T22:21:33.513Z 13 260-279 126 126 126 239 239 239 T T F <---
140052024-02-06T22:21:33.513Z 14 280-299 126 126 126 239 239 239 T T F <---
140062024-02-06T22:21:33.513Z 15 300-319 128 128 126 243 243 240 T T F <---
140072024-02-06T22:21:33.513Z 16 320-339 124 124 124 235 235 235 F F F
140082024-02-06T22:21:33.513Z 17 340-359 128 128 128 242 242 242 F F F
140092024-02-06T22:21:33.513Z 18 360-379 122 122 122 228 228 228 F F F
140102024-02-06T22:21:33.513Z 19 380-399 118 118 118 222 222 222 F F F
140112024-02-06T22:21:33.513Z 20 400-419 123 123 123 231 231 231 F F F
140122024-02-06T22:21:33.513Z 21 420-439 127 127 127 241 241 241 F F F
140132024-02-06T22:21:33.513Z 22 440-459 126 126 126 237 237 237 F F F
140142024-02-06T22:21:33.513Z 23 460-479 122 122 122 230 230 230 F F F
140152024-02-06T22:21:33.513Z 24 480-499 126 126 126 240 240 240 F F F
140162024-02-06T22:21:33.513Z 25 500-519 125 125 125 236 236 236 F F F
140172024-02-06T22:21:33.514Z 26 520-539 128 128 127 243 243 241 F F T <---
140182024-02-06T22:21:33.514Z 27 540-559 126 126 126 239 239 239 F F F
140192024-02-06T22:21:33.514Z 28 560-579 126 126 126 240 240 240 F F F
140202024-02-06T22:21:33.514Z 29 580-599 127 127 127 241 241 241 F F F
140212024-02-06T22:21:33.514ZMax gen: 128, Max flush: 243
140222024-02-06T22:21:33.514ZError: Difference in extent metadata found!
140232024-02-06T22:21:33.514ZA Difference in extent metadata is expected here
140242024-02-06T22:21:33.514ZOn loop 64
140252024-02-06T22:21:33.514Z
140262024-02-06T22:21:33.520ZVerifying data now
140272024-02-06T22:21:39.078ZNo change after verify
140282024-02-06T22:21:39.078ZLoop: 64 Downstairs dump after verify (and repair):
140292024-02-06T22:21:39.093ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140302024-02-06T22:21:39.093Z 0 000-019 122 122 122 227 227 227 F F F
140312024-02-06T22:21:39.093Z 1 020-039 128 128 128 243 243 243 F F F
140322024-02-06T22:21:39.093Z 2 040-059 128 128 128 243 243 243 F F F
140332024-02-06T22:21:39.093Z 3 060-079 109 109 109 205 205 205 F F F
140342024-02-06T22:21:39.093Z 4 080-099 115 115 115 217 217 217 F F F
140352024-02-06T22:21:39.093Z 5 100-119 129 129 129 244 244 244 F F F
140362024-02-06T22:21:39.093Z 6 120-139 129 129 129 244 244 244 F F F
140372024-02-06T22:21:39.093Z 7 140-159 129 129 129 244 244 244 F F F
140382024-02-06T22:21:39.093Z 8 160-179 128 128 128 243 243 243 F F F
140392024-02-06T22:21:39.093Z 9 180-199 128 128 128 242 242 242 F F F
140402024-02-06T22:21:39.094Z 10 200-219 126 126 126 239 239 239 F F F
140412024-02-06T22:21:39.094Z 11 220-239 128 128 128 243 243 243 F F F
140422024-02-06T22:21:39.097Z 12 240-259 126 126 126 239 239 239 F F F
140432024-02-06T22:21:39.097Z 13 260-279 129 129 129 244 244 244 F F F
140442024-02-06T22:21:39.097Z 14 280-299 129 129 129 244 244 244 F F F
140452024-02-06T22:21:39.097Z 15 300-319 129 129 129 244 244 244 F F F
140462024-02-06T22:21:39.097Z 16 320-339 124 124 124 235 235 235 F F F
140472024-02-06T22:21:39.097Z 17 340-359 128 128 128 242 242 242 F F F
140482024-02-06T22:21:39.097Z 18 360-379 122 122 122 228 228 228 F F F
140492024-02-06T22:21:39.097Z 19 380-399 118 118 118 222 222 222 F F F
140502024-02-06T22:21:39.097Z 20 400-419 123 123 123 231 231 231 F F F
140512024-02-06T22:21:39.097Z 21 420-439 127 127 127 241 241 241 F F F
140522024-02-06T22:21:39.097Z 22 440-459 126 126 126 237 237 237 F F F
140532024-02-06T22:21:39.097Z 23 460-479 122 122 122 230 230 230 F F F
140542024-02-06T22:21:39.097Z 24 480-499 126 126 126 240 240 240 F F F
140552024-02-06T22:21:39.097Z 25 500-519 125 125 125 236 236 236 F F F
140562024-02-06T22:21:39.097Z 26 520-539 128 128 128 243 243 243 F F F
140572024-02-06T22:21:39.097Z 27 540-559 126 126 126 239 239 239 F F F
140582024-02-06T22:21:39.097Z 28 560-579 126 126 126 240 240 240 F F F
140592024-02-06T22:21:39.097Z 29 580-599 127 127 127 241 241 241 F F F
140602024-02-06T22:21:39.098ZMax gen: 129, Max flush: 244
140612024-02-06T22:21:39.098Z
140622024-02-06T22:21:39.098ZBegin loop 65
140632024-02-06T22:21:39.098ZDownstairs to restart: 1
140642024-02-06T22:21:39.117Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.307598104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658}
140652024-02-06T22:21:39.123Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.307902343Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658}
140662024-02-06T22:21:39.123ZCrucible runtime is spawned
140672024-02-06T22:21:39.123Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.307946817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658}
140682024-02-06T22:21:39.124Z{"msg":"Crucible 616f5852-8e1f-4548-a7c1-03fca96158d7 has session id: 6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.307977673Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140692024-02-06T22:21:39.124Z{"msg":"Upstairs opts: Upstairs UUID: 616f5852-8e1f-4548-a7c1-03fca96158d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308033865Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140702024-02-06T22:21:39.124Z{"msg":"Crucible stats registered with UUID: 616f5852-8e1f-4548-a7c1-03fca96158d7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.30807715Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140712024-02-06T22:21:39.124Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308122824Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658}
140722024-02-06T22:21:39.124Z{"msg":"616f5852-8e1f-4548-a7c1-03fca96158d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.30815023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140732024-02-06T22:21:39.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308183135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140742024-02-06T22:21:39.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308214041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140752024-02-06T22:21:39.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308263025Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140762024-02-06T22:21:39.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308292061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140772024-02-06T22:21:39.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308337844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140782024-02-06T22:21:39.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308451439Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140792024-02-06T22:21:39.124Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308544936Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140802024-02-06T22:21:39.124Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308607708Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140812024-02-06T22:21:39.124Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308630745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140822024-02-06T22:21:39.124Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308654072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140832024-02-06T22:21:39.124Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.30866885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140842024-02-06T22:21:39.124Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.308690657Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140852024-02-06T22:21:39.124Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.309217146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140862024-02-06T22:21:39.124Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.309254161Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140872024-02-06T22:21:39.124Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.309270438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140882024-02-06T22:21:39.124Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.309299674Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140892024-02-06T22:21:39.124Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.309941228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140902024-02-06T22:21:39.125Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.30999908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140912024-02-06T22:21:39.125Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.310471516Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140922024-02-06T22:21:39.125Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.31051891Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140932024-02-06T22:21:39.125Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.310561044Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140942024-02-06T22:21:39.125Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.310634074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140952024-02-06T22:21:39.125Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.310671759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140962024-02-06T22:21:39.125Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.310688127Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140972024-02-06T22:21:39.125Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.310709544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140982024-02-06T22:21:39.125Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.310723432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
140992024-02-06T22:21:39.125Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311192899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141002024-02-06T22:21:39.125Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311620261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141012024-02-06T22:21:39.125Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311651697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141022024-02-06T22:21:39.125Z{"msg":"[0]R flush_numbers[0..12]: [227, 243, 243, 205, 217, 244, 244, 244, 243, 242, 239, 243]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311738585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141032024-02-06T22:21:39.125Z{"msg":"[0]R generation[0..12]: [122, 128, 128, 109, 115, 129, 129, 129, 128, 128, 126, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311764821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141042024-02-06T22:21:39.125Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311780979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141052024-02-06T22:21:39.125Z{"msg":"[1]R flush_numbers[0..12]: [227, 243, 243, 205, 217, 244, 244, 244, 243, 242, 239, 243]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311803076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141062024-02-06T22:21:39.125Z{"msg":"[1]R generation[0..12]: [122, 128, 128, 109, 115, 129, 129, 129, 128, 128, 126, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311817474Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141072024-02-06T22:21:39.125Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311838951Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141082024-02-06T22:21:39.125Z{"msg":"[2]R flush_numbers[0..12]: [227, 243, 243, 205, 217, 244, 244, 244, 243, 242, 239, 243]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311853959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141092024-02-06T22:21:39.125ZWait for a query_work_queue command to finish before sending IO
141102024-02-06T22:21:39.125Z{"msg":"[2]R generation[0..12]: [122, 128, 128, 109, 115, 129, 129, 129, 128, 128, 126, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311877566Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141112024-02-06T22:21:39.125Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311892954Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141122024-02-06T22:21:39.125Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311906742Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141132024-02-06T22:21:39.125Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311932279Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141142024-02-06T22:21:39.125Z{"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311945757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141152024-02-06T22:21:39.126Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311959435Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141162024-02-06T22:21:39.126Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311974003Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141172024-02-06T22:21:39.126Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.311986331Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141182024-02-06T22:21:39.126Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.31200031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141192024-02-06T22:21:39.126Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.312013868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141202024-02-06T22:21:39.126Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.312027886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141212024-02-06T22:21:39.126Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.312047273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141222024-02-06T22:21:39.126Z{"msg":"616f5852-8e1f-4548-a7c1-03fca96158d7 is now active with session: 6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.312060481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141232024-02-06T22:21:39.126Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.312079609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658,"session_id":"6f9aa7b3-f2e0-4bd6-bc0f-31fd7e830c8e"}
141242024-02-06T22:21:39.126Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:38.312092177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658}
141252024-02-06T22:21:39.126ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
141262024-02-06T22:21:39.126ZLoading write count information from file "/tmp/test_repair_verify.data"
141272024-02-06T22:21:39.126ZRun Repair workload
141282024-02-06T22:21:39.126Z01/30 Write block 420 len 12288 data: 13 11 9
141292024-02-06T22:21:39.129Z02/30 Write block 356 len 12288 data: 16 17 15
141302024-02-06T22:21:39.130Z03/30 Flush
141312024-02-06T22:21:39.130Z04/30 Write block 418 len 20480 data: 15 14 14 12 10
141322024-02-06T22:21:39.130Z05/30 Write block 530 len 24576 data: 13 14 12 15 12 12
141332024-02-06T22:21:39.130Z06/30 Read block 378 len 16384
141342024-02-06T22:21:39.130Z07/30 Read block 513 len 8192
141352024-02-06T22:21:39.130Z08/30 Write block 405 len 4096 data: 12
141362024-02-06T22:21:39.130Z09/30 Write block 11 len 40960 data: 7 7 6 8 9 8 7 8 7 6
141372024-02-06T22:21:39.130Z10/30 Flush
141382024-02-06T22:21:39.133Z11/30 Write block 205 len 4096 data: 12
141392024-02-06T22:21:39.133Z12/30 Read block 258 len 36864
141402024-02-06T22:21:39.136Z13/30 Write block 537 len 36864 data: 9 8 6 7 8 8 9 11 9
141412024-02-06T22:21:39.136Z14/30 Read block 273 len 16384
141422024-02-06T22:21:39.136Z15/30 Read block 149 len 20480
141432024-02-06T22:21:39.136Z16/30 Write block 373 len 4096 data: 9
141442024-02-06T22:21:39.139Z17/30 Read block 554 len 16384
141452024-02-06T22:21:39.139Z18/30 Read block 474 len 20480
141462024-02-06T22:21:39.139Z19/30 Read block 137 len 20480
141472024-02-06T22:21:39.139Z20/30 Read block 311 len 12288
141482024-02-06T22:21:39.142Z21/30 Read block 80 len 20480
141492024-02-06T22:21:39.142Z22/30 Write block 454 len 4096 data: 12
141502024-02-06T22:21:39.142Z23/30 Read block 82 len 8192
141512024-02-06T22:21:39.142Z24/30 Read block 371 len 28672
141522024-02-06T22:21:39.145Z25/30 Write block 322 len 16384 data: 12 10 10 9
141532024-02-06T22:21:39.145Z26/30 Read block 275 len 16384
141542024-02-06T22:21:39.145Z27/30 Read block 411 len 40960
141552024-02-06T22:21:39.145Z28/30 Read block 240 len 36864
141562024-02-06T22:21:39.149Z29/30 Read block 366 len 28672
141572024-02-06T22:21:39.149Z30/30 Read block 349 len 32768
141582024-02-06T22:21:39.149Z----------------------------------------------------------------
141592024-02-06T22:21:39.149Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
141602024-02-06T22:21:39.149ZStates: Active Active Active
141612024-02-06T22:21:39.149ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
141622024-02-06T22:21:39.149Z 1 Acked 1000 Write 3 Done Done Done false
141632024-02-06T22:21:39.149Z 2 Acked 1001 Write 3 Done Done Done false
141642024-02-06T22:21:39.149Z 3 Acked 1002 Flush 0 Done Sent Done false
141652024-02-06T22:21:39.149Z 4 Acked 1003 Write 5 Done Sent Done false
141662024-02-06T22:21:39.149Z 5 Acked 1004 Write 6 Done Sent Done false
141672024-02-06T22:21:39.150Z 6 Acked 1005 Read 4 Done Sent Done false
141682024-02-06T22:21:39.150Z 7 Acked 1006 Read 2 Done Sent Done false
141692024-02-06T22:21:39.150Z 8 Acked 1007 Write 1 Done Sent Done false
141702024-02-06T22:21:39.150Z 9 Acked 1008 Write 10 Done Sent Done false
141712024-02-06T22:21:39.150Z 10 Acked 1009 Flush 0 Done Sent Done false
141722024-02-06T22:21:39.150Z 11 Acked 1010 Write 1 Done Sent Done false
141732024-02-06T22:21:39.150Z 12 Acked 1011 Read 9 Done Sent Done false
141742024-02-06T22:21:39.150Z 13 Acked 1012 Write 9 Done Sent Done false
141752024-02-06T22:21:39.150Z 14 Acked 1013 Read 4 Done Sent Done false
141762024-02-06T22:21:39.150Z 15 Acked 1014 Read 5 Done Sent Done false
141772024-02-06T22:21:39.150Z 16 Acked 1015 Write 1 Done Sent Done false
141782024-02-06T22:21:39.150Z 17 Acked 1016 Read 4 Done Sent Done false
141792024-02-06T22:21:39.150Z 18 Acked 1017 Read 5 Done Sent Done false
141802024-02-06T22:21:39.150Z 19 Acked 1018 Read 5 Done Sent Done false
141812024-02-06T22:21:39.150Z 20 Acked 1019 Read 3 Done Sent Done false
141822024-02-06T22:21:39.150Z 21 Acked 1020 Read 5 Done Sent Done false
141832024-02-06T22:21:39.150Z 22 Acked 1021 Write 1 Done Sent Done false
141842024-02-06T22:21:39.150Z 23 Acked 1022 Read 2 Done Sent Done false
141852024-02-06T22:21:39.150Z 24 Acked 1023 Read 7 Done Sent Done false
141862024-02-06T22:21:39.150Z 25 Acked 1024 Write 4 Done Sent Done false
141872024-02-06T22:21:39.150Z 26 Acked 1025 Read 4 Done Sent Done false
141882024-02-06T22:21:39.150Z 27 Acked 1026 Read 10 Done Sent Done false
141892024-02-06T22:21:39.150Z 28 Acked 1027 Read 9 Done Sent Done false
141902024-02-06T22:21:39.150Z 29 Acked 1028 Read 7 Done Sent Done false
141912024-02-06T22:21:39.151Z 30 Acked 1029 Read 8 Done Sent Sent false
141922024-02-06T22:21:39.151Z STATES DS:0 DS:1 DS:2 TOTAL
141932024-02-06T22:21:39.151Z New 0 0 0 0
141942024-02-06T22:21:39.151Z Sent 0 28 1 29
141952024-02-06T22:21:39.151Z Done 30 2 29 61
141962024-02-06T22:21:39.151Z Skipped 0 0 0 0
141972024-02-06T22:21:39.151Z Error 0 0 0 0
141982024-02-06T22:21:39.151ZLast Flush: 1009 0 1009
141992024-02-06T22:21:39.151ZDownstairs last five completed:
142002024-02-06T22:21:39.151ZUpstairs last five completed: 30 29 28 27 26
142012024-02-06T22:21:39.151Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:38.340927711Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1658}
142022024-02-06T22:21:39.151ZWrote out file "/tmp/test_repair_verify.data"
142032024-02-06T22:21:39.154Z
142042024-02-06T22:21:39.157ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142052024-02-06T22:21:39.172ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142062024-02-06T22:21:39.172Z 0 000-019 130 122 130 246 227 246 F F F <---
142072024-02-06T22:21:39.172Z 1 020-039 130 128 130 246 243 246 F F F <---
142082024-02-06T22:21:39.173Z 2 040-059 128 128 128 243 243 243 F F F
142092024-02-06T22:21:39.173Z 3 060-079 109 109 109 205 205 205 F F F
142102024-02-06T22:21:39.173Z 4 080-099 115 115 115 217 217 217 F F F
142112024-02-06T22:21:39.173Z 5 100-119 129 129 129 244 244 244 F F F
142122024-02-06T22:21:39.173Z 6 120-139 129 129 129 244 244 244 F F F
142132024-02-06T22:21:39.173Z 7 140-159 129 129 129 244 244 244 F F F
142142024-02-06T22:21:39.173Z 8 160-179 128 128 128 243 243 243 F F F
142152024-02-06T22:21:39.173Z 9 180-199 128 128 128 242 242 242 F F F
142162024-02-06T22:21:39.173Z 10 200-219 126 126 126 239 239 239 T F T <---
142172024-02-06T22:21:39.173Z 11 220-239 128 128 128 243 243 243 F F F
142182024-02-06T22:21:39.173Z 12 240-259 126 126 126 239 239 239 F F F
142192024-02-06T22:21:39.173Z 13 260-279 129 129 129 244 244 244 F F F
142202024-02-06T22:21:39.176Z 14 280-299 129 129 129 244 244 244 F F F
142212024-02-06T22:21:39.176Z 15 300-319 129 129 129 244 244 244 F F F
142222024-02-06T22:21:39.176Z 16 320-339 124 124 124 235 235 235 T F T <---
142232024-02-06T22:21:39.176Z 17 340-359 130 128 130 245 242 245 F T F <---
142242024-02-06T22:21:39.176Z 18 360-379 122 122 122 228 228 228 T F T <---
142252024-02-06T22:21:39.176Z 19 380-399 118 118 118 222 222 222 F F F
142262024-02-06T22:21:39.176Z 20 400-419 130 123 130 246 231 246 F F F <---
142272024-02-06T22:21:39.176Z 21 420-439 130 127 130 246 241 246 F T F <---
142282024-02-06T22:21:39.177Z 22 440-459 126 126 126 237 237 237 T F T <---
142292024-02-06T22:21:39.177Z 23 460-479 122 122 122 230 230 230 F F F
142302024-02-06T22:21:39.177Z 24 480-499 126 126 126 240 240 240 F F F
142312024-02-06T22:21:39.177Z 25 500-519 125 125 125 236 236 236 F F F
142322024-02-06T22:21:39.177Z 26 520-539 130 128 130 246 243 246 T F T <---
142332024-02-06T22:21:39.177Z 27 540-559 126 126 126 239 239 239 T F T <---
142342024-02-06T22:21:39.177Z 28 560-579 126 126 126 240 240 240 F F F
142352024-02-06T22:21:39.177Z 29 580-599 127 127 127 241 241 241 F F F
142362024-02-06T22:21:39.177ZMax gen: 130, Max flush: 246
142372024-02-06T22:21:39.177ZError: Difference in extent metadata found!
142382024-02-06T22:21:39.177ZA Difference in extent metadata is expected here
142392024-02-06T22:21:39.177ZOn loop 65
142402024-02-06T22:21:39.177Z
142412024-02-06T22:21:39.180ZVerifying data now
142422024-02-06T22:21:44.743ZNo change after verify
142432024-02-06T22:21:44.743ZLoop: 65 Downstairs dump after verify (and repair):
142442024-02-06T22:21:44.758ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142452024-02-06T22:21:44.758Z 0 000-019 130 130 130 246 246 246 F F F
142462024-02-06T22:21:44.758Z 1 020-039 130 130 130 246 246 246 F F F
142472024-02-06T22:21:44.758Z 2 040-059 128 128 128 243 243 243 F F F
142482024-02-06T22:21:44.758Z 3 060-079 109 109 109 205 205 205 F F F
142492024-02-06T22:21:44.758Z 4 080-099 115 115 115 217 217 217 F F F
142502024-02-06T22:21:44.758Z 5 100-119 129 129 129 244 244 244 F F F
142512024-02-06T22:21:44.758Z 6 120-139 129 129 129 244 244 244 F F F
142522024-02-06T22:21:44.758Z 7 140-159 129 129 129 244 244 244 F F F
142532024-02-06T22:21:44.758Z 8 160-179 128 128 128 243 243 243 F F F
142542024-02-06T22:21:44.758Z 9 180-199 128 128 128 242 242 242 F F F
142552024-02-06T22:21:44.758Z 10 200-219 131 131 131 247 247 247 F F F
142562024-02-06T22:21:44.759Z 11 220-239 128 128 128 243 243 243 F F F
142572024-02-06T22:21:44.762Z 12 240-259 126 126 126 239 239 239 F F F
142582024-02-06T22:21:44.762Z 13 260-279 129 129 129 244 244 244 F F F
142592024-02-06T22:21:44.762Z 14 280-299 129 129 129 244 244 244 F F F
142602024-02-06T22:21:44.762Z 15 300-319 129 129 129 244 244 244 F F F
142612024-02-06T22:21:44.762Z 16 320-339 131 131 131 247 247 247 F F F
142622024-02-06T22:21:44.762Z 17 340-359 130 130 130 245 245 245 F F F
142632024-02-06T22:21:44.762Z 18 360-379 131 131 131 247 247 247 F F F
142642024-02-06T22:21:44.762Z 19 380-399 118 118 118 222 222 222 F F F
142652024-02-06T22:21:44.762Z 20 400-419 130 130 130 246 246 246 F F F
142662024-02-06T22:21:44.763Z 21 420-439 130 130 130 246 246 246 F F F
142672024-02-06T22:21:44.763Z 22 440-459 131 131 131 247 247 247 F F F
142682024-02-06T22:21:44.763Z 23 460-479 122 122 122 230 230 230 F F F
142692024-02-06T22:21:44.763Z 24 480-499 126 126 126 240 240 240 F F F
142702024-02-06T22:21:44.763Z 25 500-519 125 125 125 236 236 236 F F F
142712024-02-06T22:21:44.763Z 26 520-539 131 131 131 247 247 247 F F F
142722024-02-06T22:21:44.763Z 27 540-559 131 131 131 247 247 247 F F F
142732024-02-06T22:21:44.763Z 28 560-579 126 126 126 240 240 240 F F F
142742024-02-06T22:21:44.763Z 29 580-599 127 127 127 241 241 241 F F F
142752024-02-06T22:21:44.763ZMax gen: 131, Max flush: 247
142762024-02-06T22:21:44.763Z
142772024-02-06T22:21:44.763ZBegin loop 66
142782024-02-06T22:21:44.763ZDownstairs to restart: 2
142792024-02-06T22:21:44.789Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.97070547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666}
142802024-02-06T22:21:44.789Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.970977813Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666}
142812024-02-06T22:21:44.789ZCrucible runtime is spawned
142822024-02-06T22:21:44.796Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971026697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666}
142832024-02-06T22:21:44.796Z{"msg":"Crucible 76d0f416-2c89-44e6-8bfa-09609a087121 has session id: 40a88b5c-d45d-4fc3-aafd-b3f143b3a901","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971064192Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142842024-02-06T22:21:44.796Z{"msg":"Upstairs opts: Upstairs UUID: 76d0f416-2c89-44e6-8bfa-09609a087121, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971101916Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142852024-02-06T22:21:44.796Z{"msg":"Crucible stats registered with UUID: 76d0f416-2c89-44e6-8bfa-09609a087121","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971136852Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142862024-02-06T22:21:44.796Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971170707Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666}
142872024-02-06T22:21:44.796Z{"msg":"76d0f416-2c89-44e6-8bfa-09609a087121 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971275413Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142882024-02-06T22:21:44.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971313178Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142892024-02-06T22:21:44.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971340954Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142902024-02-06T22:21:44.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.97137266Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142912024-02-06T22:21:44.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971399226Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142922024-02-06T22:21:44.796Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971430992Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142932024-02-06T22:21:44.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971497013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142942024-02-06T22:21:44.796Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971578392Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142952024-02-06T22:21:44.796Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971641983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142962024-02-06T22:21:44.796Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971691637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142972024-02-06T22:21:44.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971751049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142982024-02-06T22:21:44.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971789263Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
142992024-02-06T22:21:44.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.971807061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143002024-02-06T22:21:44.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.972297135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143012024-02-06T22:21:44.797Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.972326681Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143022024-02-06T22:21:44.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.972342829Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143032024-02-06T22:21:44.797Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.972371835Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143042024-02-06T22:21:44.797Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.972876107Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143052024-02-06T22:21:44.797Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973063301Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143062024-02-06T22:21:44.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973348753Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143072024-02-06T22:21:44.797Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973404775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143082024-02-06T22:21:44.797Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973468007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143092024-02-06T22:21:44.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973487164Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143102024-02-06T22:21:44.797Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973513661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143112024-02-06T22:21:44.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973605908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143122024-02-06T22:21:44.797Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973634264Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143132024-02-06T22:21:44.797Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973648872Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143142024-02-06T22:21:44.797Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.973992626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143152024-02-06T22:21:44.797Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974458173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143162024-02-06T22:21:44.797Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974496888Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143172024-02-06T22:21:44.797Z{"msg":"[0]R flush_numbers[0..12]: [246, 246, 243, 205, 217, 244, 244, 244, 243, 242, 247, 243]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974524244Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143182024-02-06T22:21:44.797Z{"msg":"[0]R generation[0..12]: [130, 130, 128, 109, 115, 129, 129, 129, 128, 128, 131, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.97455435Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143192024-02-06T22:21:44.797Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
143202024-02-06T22:21:44.797Z,"time":"2024-02-06T22:21:43.974580936Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143212024-02-06T22:21:44.797Z{"msg":"[1]R flush_numbers[0..12]: [246, 246, 243, 205, 217, 244, 244, 244, 243, 242, 247, 243]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974613012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143222024-02-06T22:21:44.797Z{"msg":"[1]R generation[0..12]: [130, 130, 128, 109, 115, 129, 129, 129, 128, 128, 131, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974646968Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143232024-02-06T22:21:44.797Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974674044Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143242024-02-06T22:21:44.798Z{"msg":"[2]R flush_numbers[0..12]: [246, 246, 243, 205, 217, 244, 244, 244, 243, 242, 247, 243]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.97470406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143252024-02-06T22:21:44.798Z{"msg":"[2]R generation[0..12]: [130, 130, 128, 109, 115, 129, 129, 129, 128, 128, 131, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974729816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143262024-02-06T22:21:44.798Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974761422Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143272024-02-06T22:21:44.798Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974785369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143282024-02-06T22:21:44.798Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974808835Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143292024-02-06T22:21:44.798Z{"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974839761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143302024-02-06T22:21:44.798Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974863768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143312024-02-06T22:21:44.798Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974891155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143322024-02-06T22:21:44.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
143332024-02-06T22:21:44.798Z,"time":"2024-02-06T22:21:43.974913361Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143342024-02-06T22:21:44.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974948247Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143352024-02-06T22:21:44.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.974980472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143362024-02-06T22:21:44.798Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.975012188Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143372024-02-06T22:21:44.798Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.975034575Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143382024-02-06T22:21:44.798Z{"msg":"76d0f416-2c89-44e6-8bfa-09609a087121 is now active with session: 40a88b5c-d45d-4fc3-aafd-b3f143b3a901","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.975066381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143392024-02-06T22:21:44.798Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.975096797Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666,"session_id":"40a88b5c-d45d-4fc3-aafd-b3f143b3a901"}
143402024-02-06T22:21:44.798Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:43.975118394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666}
143412024-02-06T22:21:44.798ZLoading write count information from file "/tmp/test_repair_verify.data"
143422024-02-06T22:21:44.798ZRun Repair workload
143432024-02-06T22:21:44.798Z01/30 Write block 315 len 32768 data: 16 15 14 13 11 9 10 13
143442024-02-06T22:21:44.798Z02/30 Read block 512 len 16384
143452024-02-06T22:21:44.798Z03/30 Write block 213 len 8192 data: 13 15
143462024-02-06T22:21:44.798Z04/30 Write block 106 len 24576 data: 11 12 15 11 12 11
143472024-02-06T22:21:44.798Z05/30 Flush
143482024-02-06T22:21:44.798Z06/30 Write block 71 len 12288 data: 8 9 9
143492024-02-06T22:21:44.799Z07/30 Read block 493 len 4096
143502024-02-06T22:21:44.799Z08/30 Write block 97 len 20480 data: 8 9 9 10 7
143512024-02-06T22:21:44.799Z09/30 Read block 30 len 20480
143522024-02-06T22:21:44.809Z10/30 Read block 116 len 36864
143532024-02-06T22:21:44.809Z11/30 Write block 344 len 40960 data: 13 10 12 12 14 14 15 19 20 18
143542024-02-06T22:21:44.809Z12/30 Write block 225 len 28672 data: 11 11 8 8 8 8 11
143552024-02-06T22:21:44.810Z13/30 Read block 551 len 40960
143562024-02-06T22:21:44.810Z14/30 Write block 352 len 12288 data: 21 19 17
143572024-02-06T22:21:44.810Z15/30 Read block 448 len 16384
143582024-02-06T22:21:44.810Z16/30 Read block 82 len 32768
143592024-02-06T22:21:44.810Z17/30 Read block 349 len 16384
143602024-02-06T22:21:44.810Z18/30 Read block 411 len 12288
143612024-02-06T22:21:44.810Z19/30 Read block 68 len 36864
143622024-02-06T22:21:44.810Z20/30 Read block 381 len 16384
143632024-02-06T22:21:44.810Z21/30 Read block 37 len 4096
143642024-02-06T22:21:44.810Z22/30 Write block 121 len 20480 data: 10 11 12 12 13
143652024-02-06T22:21:44.810Z23/30 Read block 450 len 36864
143662024-02-06T22:21:44.810Z24/30 Read block 389 len 12288
143672024-02-06T22:21:44.810Z25/30 Write block 150 len 40960 data: 14 12 13 15 14 16 13 15 11 12
143682024-02-06T22:21:44.810Z26/30 Read block 335 len 4096
143692024-02-06T22:21:44.810Z27/30 Read block 414 len 24576
143702024-02-06T22:21:44.814Z28/30 Read block 444 len 36864
143712024-02-06T22:21:44.815Z29/30 Read block 354 len 8192
143722024-02-06T22:21:44.815Z30/30 Read block 16 len 24576
143732024-02-06T22:21:44.815Z----------------------------------------------------------------
143742024-02-06T22:21:44.815Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:25
143752024-02-06T22:21:44.815ZStates: Active Active Active
143762024-02-06T22:21:44.815ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
143772024-02-06T22:21:44.815Z 6 Acked 1005 Write 3 Done Done Done false
143782024-02-06T22:21:44.815Z 7 Acked 1006 Read 1 Done Done Done false
143792024-02-06T22:21:44.815Z 8 Acked 1007 Write 5 Done Done Done false
143802024-02-06T22:21:44.815Z 9 Acked 1008 Read 5 Done Done Done false
143812024-02-06T22:21:44.815Z 10 Acked 1009 Read 9 Done Done Done false
143822024-02-06T22:21:44.815Z 11 Acked 1010 Write 10 Done Done Done false
143832024-02-06T22:21:44.815Z 12 Acked 1011 Write 7 Done Done Done false
143842024-02-06T22:21:44.815Z 13 Acked 1012 Read 10 Done Done Done false
143852024-02-06T22:21:44.815Z 14 Acked 1013 Write 3 Done Done Sent false
143862024-02-06T22:21:44.815Z 15 Acked 1014 Read 4 Done Done Sent false
143872024-02-06T22:21:44.815Z 16 Acked 1015 Read 8 Done Done Sent false
143882024-02-06T22:21:44.815Z 17 Acked 1016 Read 4 Done Done Sent false
143892024-02-06T22:21:44.815Z 18 Acked 1017 Read 3 Done Done Sent false
143902024-02-06T22:21:44.815Z 19 Acked 1018 Read 9 Done Done Sent false
143912024-02-06T22:21:44.815Z 20 Acked 1019 Read 4 Done Done Sent false
143922024-02-06T22:21:44.815Z 21 Acked 1020 Read 1 Done Done Sent false
143932024-02-06T22:21:44.815Z 22 Acked 1021 Write 5 Done Done Sent false
143942024-02-06T22:21:44.815Z 23 Acked 1022 Read 9 Done Done Sent false
143952024-02-06T22:21:44.816Z 24 Acked 1023 Read 3 Done Done Sent false
143962024-02-06T22:21:44.816Z 25 Acked 1024 Write 10 Done Done Sent false
143972024-02-06T22:21:44.816Z 26 Acked 1025 Read 1 Done Done Sent false
143982024-02-06T22:21:44.816Z 27 Acked 1026 Read 6 Done Done Sent false
143992024-02-06T22:21:44.816Z 28 Acked 1027 Read 9 Done Done Sent false
144002024-02-06T22:21:44.816Z 29 Acked 1028 Read 2 Done Done Sent false
144012024-02-06T22:21:44.816Z 30 Acked 1029 Read 6 Done Sent Sent false
144022024-02-06T22:21:44.816Z STATES DS:0 DS:1 DS:2 TOTAL
144032024-02-06T22:21:44.816Z New 0 0 0 0
144042024-02-06T22:21:44.816Z Sent 0 1 17 18
144052024-02-06T22:21:44.816Z Done 25 24 8 57
144062024-02-06T22:21:44.816Z Skipped 0 0 0 0
144072024-02-06T22:21:44.816Z Error 0 0 0 0
144082024-02-06T22:21:44.816ZLast Flush: 1004 1004 1004
144092024-02-06T22:21:44.816ZDownstairs last five completed: 1004 1003 1002 1001 1000
144102024-02-06T22:21:44.816ZUpstairs last five completed: 30 29 28 27 26
144112024-02-06T22:21:44.816Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:44.004747901Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1666}
144122024-02-06T22:21:44.816ZWrote out file "/tmp/test_repair_verify.data"
144132024-02-06T22:21:44.819Z
144142024-02-06T22:21:44.822ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
144152024-02-06T22:21:44.839ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144162024-02-06T22:21:44.839Z 0 000-019 130 130 130 246 246 246 F F F
144172024-02-06T22:21:44.839Z 1 020-039 130 130 130 246 246 246 F F F
144182024-02-06T22:21:44.839Z 2 040-059 128 128 128 243 243 243 F F F
144192024-02-06T22:21:44.839Z 3 060-079 109 109 109 205 205 205 T T T
144202024-02-06T22:21:44.839Z 4 080-099 115 115 115 217 217 217 T T T
144212024-02-06T22:21:44.839Z 5 100-119 132 132 132 248 248 248 T T T
144222024-02-06T22:21:44.842Z 6 120-139 129 129 129 244 244 244 T T F <---
144232024-02-06T22:21:44.842Z 7 140-159 129 129 129 244 244 244 T T F <---
144242024-02-06T22:21:44.842Z 8 160-179 128 128 128 243 243 243 F F F
144252024-02-06T22:21:44.842Z 9 180-199 128 128 128 242 242 242 F F F
144262024-02-06T22:21:44.842Z 10 200-219 132 132 132 248 248 248 F F F
144272024-02-06T22:21:44.842Z 11 220-239 128 128 128 243 243 243 T T T
144282024-02-06T22:21:44.842Z 12 240-259 126 126 126 239 239 239 F F F
144292024-02-06T22:21:44.842Z 13 260-279 129 129 129 244 244 244 F F F
144302024-02-06T22:21:44.842Z 14 280-299 129 129 129 244 244 244 F F F
144312024-02-06T22:21:44.842Z 15 300-319 132 132 132 248 248 248 F F F
144322024-02-06T22:21:44.842Z 16 320-339 132 132 132 248 248 248 F F F
144332024-02-06T22:21:44.842Z 17 340-359 130 130 130 245 245 245 T T T
144342024-02-06T22:21:44.843Z 18 360-379 131 131 131 247 247 247 F F F
144352024-02-06T22:21:44.843Z 19 380-399 118 118 118 222 222 222 F F F
144362024-02-06T22:21:44.843Z 20 400-419 130 130 130 246 246 246 F F F
144372024-02-06T22:21:44.843Z 21 420-439 130 130 130 246 246 246 F F F
144382024-02-06T22:21:44.843Z 22 440-459 131 131 131 247 247 247 F F F
144392024-02-06T22:21:44.843Z 23 460-479 122 122 122 230 230 230 F F F
144402024-02-06T22:21:44.843Z 24 480-499 126 126 126 240 240 240 F F F
144412024-02-06T22:21:44.843Z 25 500-519 125 125 125 236 236 236 F F F
144422024-02-06T22:21:44.843Z 26 520-539 131 131 131 247 247 247 F F F
144432024-02-06T22:21:44.843Z 27 540-559 131 131 131 247 247 247 F F F
144442024-02-06T22:21:44.843Z 28 560-579 126 126 126 240 240 240 F F F
144452024-02-06T22:21:44.843Z 29 580-599 127 127 127 241 241 241 F F F
144462024-02-06T22:21:44.843ZMax gen: 132, Max flush: 248
144472024-02-06T22:21:44.843ZError: Difference in extent metadata found!
144482024-02-06T22:21:44.843ZA Difference in extent metadata is expected here
144492024-02-06T22:21:44.843ZOn loop 66
144502024-02-06T22:21:44.843Z
144512024-02-06T22:21:44.846ZVerifying data now
144522024-02-06T22:21:50.251ZNo change after verify
144532024-02-06T22:21:50.251ZLoop: 66 Downstairs dump after verify (and repair):
144542024-02-06T22:21:50.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144552024-02-06T22:21:50.267Z 0 000-019 130 130 130 246 246 246 F F F
144562024-02-06T22:21:50.267Z 1 020-039 130 130 130 246 246 246 F F F
144572024-02-06T22:21:50.267Z 2 040-059 128 128 128 243 243 243 F F F
144582024-02-06T22:21:50.267Z 3 060-079 133 133 133 249 249 249 F F F
144592024-02-06T22:21:50.267Z 4 080-099 133 133 133 249 249 249 F F F
144602024-02-06T22:21:50.267Z 5 100-119 133 133 133 249 249 249 F F F
144612024-02-06T22:21:50.267Z 6 120-139 133 133 133 249 249 249 F F F
144622024-02-06T22:21:50.267Z 7 140-159 133 133 133 249 249 249 F F F
144632024-02-06T22:21:50.267Z 8 160-179 128 128 128 243 243 243 F F F
144642024-02-06T22:21:50.267Z 9 180-199 128 128 128 242 242 242 F F F
144652024-02-06T22:21:50.267Z 10 200-219 132 132 132 248 248 248 F F F
144662024-02-06T22:21:50.267Z 11 220-239 133 133 133 249 249 249 F F F
144672024-02-06T22:21:50.267Z 12 240-259 126 126 126 239 239 239 F F F
144682024-02-06T22:21:50.267Z 13 260-279 129 129 129 244 244 244 F F F
144692024-02-06T22:21:50.270Z 14 280-299 129 129 129 244 244 244 F F F
144702024-02-06T22:21:50.271Z 15 300-319 132 132 132 248 248 248 F F F
144712024-02-06T22:21:50.271Z 16 320-339 132 132 132 248 248 248 F F F
144722024-02-06T22:21:50.271Z 17 340-359 133 133 133 249 249 249 F F F
144732024-02-06T22:21:50.271Z 18 360-379 131 131 131 247 247 247 F F F
144742024-02-06T22:21:50.271Z 19 380-399 118 118 118 222 222 222 F F F
144752024-02-06T22:21:50.271Z 20 400-419 130 130 130 246 246 246 F F F
144762024-02-06T22:21:50.271Z 21 420-439 130 130 130 246 246 246 F F F
144772024-02-06T22:21:50.271Z 22 440-459 131 131 131 247 247 247 F F F
144782024-02-06T22:21:50.271Z 23 460-479 122 122 122 230 230 230 F F F
144792024-02-06T22:21:50.271Z 24 480-499 126 126 126 240 240 240 F F F
144802024-02-06T22:21:50.271Z 25 500-519 125 125 125 236 236 236 F F F
144812024-02-06T22:21:50.271Z 26 520-539 131 131 131 247 247 247 F F F
144822024-02-06T22:21:50.271Z 27 540-559 131 131 131 247 247 247 F F F
144832024-02-06T22:21:50.271Z 28 560-579 126 126 126 240 240 240 F F F
144842024-02-06T22:21:50.271Z 29 580-599 127 127 127 241 241 241 F F F
144852024-02-06T22:21:50.271ZMax gen: 133, Max flush: 249
144862024-02-06T22:21:50.271Z
144872024-02-06T22:21:50.271ZBegin loop 67
144882024-02-06T22:21:50.271ZDownstairs to restart: 1
144892024-02-06T22:21:50.289Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480231145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674}
144902024-02-06T22:21:50.289ZCrucible runtime is spawned
144912024-02-06T22:21:50.294Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480575639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674}
144922024-02-06T22:21:50.295Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480627242Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674}
144932024-02-06T22:21:50.295Z{"msg":"Crucible 96c0487e-01a2-4d37-892f-e908abe78d7c has session id: 23dc7652-b627-45d8-b982-e1f2035130c9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480682004Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
144942024-02-06T22:21:50.295Z{"msg":"Upstairs opts: Upstairs UUID: 96c0487e-01a2-4d37-892f-e908abe78d7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480740747Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
144952024-02-06T22:21:50.295Z{"msg":"Crucible stats registered with UUID: 96c0487e-01a2-4d37-892f-e908abe78d7c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480762774Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
144962024-02-06T22:21:50.295Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48078859Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674}
144972024-02-06T22:21:50.295Z{"msg":"96c0487e-01a2-4d37-892f-e908abe78d7c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480812297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
144982024-02-06T22:21:50.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480852002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
144992024-02-06T22:21:50.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.480944519Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145002024-02-06T22:21:50.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481017779Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145012024-02-06T22:21:50.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481077931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145022024-02-06T22:21:50.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481148472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145032024-02-06T22:21:50.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481210493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145042024-02-06T22:21:50.295Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481240119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145052024-02-06T22:21:50.295Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481281963Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145062024-02-06T22:21:50.295Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48131052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145072024-02-06T22:21:50.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481334027Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145082024-02-06T22:21:50.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481364192Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145092024-02-06T22:21:50.295Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48138584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145102024-02-06T22:21:50.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481744471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145112024-02-06T22:21:50.295Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481785295Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145122024-02-06T22:21:50.295Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481855296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145132024-02-06T22:21:50.295Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.481918237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145142024-02-06T22:21:50.295Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.482386804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145152024-02-06T22:21:50.296Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.482525326Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145162024-02-06T22:21:50.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.482927971Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145172024-02-06T22:21:50.296Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.482977734Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145182024-02-06T22:21:50.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.483020719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145192024-02-06T22:21:50.296Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.483052094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145202024-02-06T22:21:50.296Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48308186Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145212024-02-06T22:21:50.296Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.483097978Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145222024-02-06T22:21:50.296Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.483315429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145232024-02-06T22:21:50.296Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.483343685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145242024-02-06T22:21:50.301Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.483808252Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145252024-02-06T22:21:50.301Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48426581Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145262024-02-06T22:21:50.301Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484300746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145272024-02-06T22:21:50.301Z{"msg":"[0]R flush_numbers[0..12]: [246, 246, 243, 249, 249, 249, 249, 249, 243, 242, 248, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484317324Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145282024-02-06T22:21:50.301Z{"msg":"[0]R generation[0..12]: [130, 130, 128, 133, 133, 133, 133, 133, 128, 128, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48434678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145292024-02-06T22:21:50.301Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484362777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145302024-02-06T22:21:50.301Z{"msg":"[1]R flush_numbers[0..12]: [246, 246, 243, 249, 249, 249, 249, 249, 243, 242, 248, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484381105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145312024-02-06T22:21:50.301Z{"msg":"[1]R generation[0..12]: [130, 130, 128, 133, 133, 133, 133, 133, 128, 128, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484402042Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145322024-02-06T22:21:50.301Z{"msg":"Wait for a query_work_queue command to finish before sending IO
145332024-02-06T22:21:50.301Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484440167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145342024-02-06T22:21:50.301Z{"msg":"[2]R flush_numbers[0..12]: [246, 246, 243, 249, 249, 249, 249, 249, 243, 242, 248, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484479351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145352024-02-06T22:21:50.301Z{"msg":"[2]R generation[0..12]: [130, 130, 128, 133, 133, 133, 133, 133, 128, 128, 132, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484510337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145362024-02-06T22:21:50.301Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484543473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145372024-02-06T22:21:50.301Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48456641Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145382024-02-06T22:21:50.301Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484604585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145392024-02-06T22:21:50.301Z{"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484625052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145402024-02-06T22:21:50.301Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484651618Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145412024-02-06T22:21:50.301Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484676205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145422024-02-06T22:21:50.302Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48471168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145432024-02-06T22:21:50.302Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484740606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145442024-02-06T22:21:50.302Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484755764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145452024-02-06T22:21:50.302Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48478505Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145462024-02-06T22:21:50.302Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484820276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145472024-02-06T22:21:50.302Z{"msg":"96c0487e-01a2-4d37-892f-e908abe78d7c is now active with session: 23dc7652-b627-45d8-b982-e1f2035130c9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484846752Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145482024-02-06T22:21:50.302Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.48486025Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674,"session_id":"23dc7652-b627-45d8-b982-e1f2035130c9"}
145492024-02-06T22:21:50.302Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:49.484889806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674}
145502024-02-06T22:21:50.302ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
145512024-02-06T22:21:50.302ZLoading write count information from file "/tmp/test_repair_verify.data"
145522024-02-06T22:21:50.302ZRun Repair workload
145532024-02-06T22:21:50.302Z01/30 Flush
145542024-02-06T22:21:50.302Z02/30 Write block 82 len 4096 data: 9
145552024-02-06T22:21:50.302Z03/30 Flush
145562024-02-06T22:21:50.302Z04/30 Write block 340 len 12288 data: 12 11 15
145572024-02-06T22:21:50.302Z05/30 Write block 205 len 12288 data: 13 12 10
145582024-02-06T22:21:50.302Z06/30 Read block 270 len 36864
145592024-02-06T22:21:50.306Z07/30 Read block 69 len 32768
145602024-02-06T22:21:50.306Z08/30 Read block 420 len 20480
145612024-02-06T22:21:50.306Z09/30 Write block 261 len 36864 data: 16 16 17 15 16 17 17 15 14
145622024-02-06T22:21:50.306Z10/30 Read block 162 len 28672
145632024-02-06T22:21:50.306Z11/30 Read block 557 len 24576
145642024-02-06T22:21:50.306Z12/30 Write block 245 len 40960 data: 8 8 13 13 15 12 9 8 8 10
145652024-02-06T22:21:50.306Z13/30 Read block 449 len 20480
145662024-02-06T22:21:50.309Z14/30 Read block 298 len 20480
145672024-02-06T22:21:50.309Z15/30 Read block 29 len 24576
145682024-02-06T22:21:50.309Z16/30 Write block 201 len 20480 data: 10 14 13 14 14
145692024-02-06T22:21:50.309Z17/30 Read block 217 len 32768
145702024-02-06T22:21:50.312Z18/30 Read block 298 len 40960
145712024-02-06T22:21:50.312Z19/30 Read block 118 len 16384
145722024-02-06T22:21:50.312Z20/30 Write block 270 len 16384 data: 12 11 11 8
145732024-02-06T22:21:50.315Z21/30 Read block 10 len 20480
145742024-02-06T22:21:50.315Z22/30 Flush
145752024-02-06T22:21:50.318Z23/30 Write block 562 len 20480 data: 10 8 10 12 12
145762024-02-06T22:21:50.318Z24/30 Read block 250 len 16384
145772024-02-06T22:21:50.318Z25/30 Read block 141 len 12288
145782024-02-06T22:21:50.321Z26/30 Write block 365 len 4096 data: 12
145792024-02-06T22:21:50.321Z27/30 Read block 12 len 28672
145802024-02-06T22:21:50.321Z28/30 Read block 245 len 40960
145812024-02-06T22:21:50.325Z29/30 Write block 363 len 32768 data: 15 13 13 10 12 12 11 10
145822024-02-06T22:21:50.325Z30/30 Read block 152 len 8192
145832024-02-06T22:21:50.325Z----------------------------------------------------------------
145842024-02-06T22:21:50.325Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:29
145852024-02-06T22:21:50.325ZStates: Active Active Active
145862024-02-06T22:21:50.325ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
145872024-02-06T22:21:50.325Z 2 Acked 1001 Write 1 Done Sent Done false
145882024-02-06T22:21:50.325Z 3 Acked 1002 Flush 0 Done Sent Done false
145892024-02-06T22:21:50.325Z 4 Acked 1003 Write 3 Done Sent Done false
145902024-02-06T22:21:50.325Z 5 Acked 1004 Write 3 Done Sent Done false
145912024-02-06T22:21:50.325Z 6 Acked 1005 Read 9 Done Sent Done false
145922024-02-06T22:21:50.325Z 7 Acked 1006 Read 8 Done Sent Done false
145932024-02-06T22:21:50.325Z 8 Acked 1007 Read 5 Done Sent Done false
145942024-02-06T22:21:50.325Z 9 Acked 1008 Write 9 Done Sent Done false
145952024-02-06T22:21:50.325Z 10 Acked 1009 Read 7 Done Sent Done false
145962024-02-06T22:21:50.325Z 11 Acked 1010 Read 6 Done Sent Done false
145972024-02-06T22:21:50.325Z 12 Acked 1011 Write 10 Done Sent Done false
145982024-02-06T22:21:50.326Z 13 Acked 1012 Read 5 Done Sent Done false
145992024-02-06T22:21:50.326Z 14 Acked 1013 Read 5 Done Sent Done false
146002024-02-06T22:21:50.326Z 15 Acked 1014 Read 6 Done Sent Done false
146012024-02-06T22:21:50.326Z 16 Acked 1015 Write 5 Done Sent Done false
146022024-02-06T22:21:50.326Z 17 Acked 1016 Read 8 Done Sent Done false
146032024-02-06T22:21:50.326Z 18 Acked 1017 Read 10 Done Sent Done false
146042024-02-06T22:21:50.326Z 19 Acked 1018 Read 4 Done Sent Done false
146052024-02-06T22:21:50.326Z 20 Acked 1019 Write 4 Done Sent Done false
146062024-02-06T22:21:50.326Z 21 Acked 1020 Read 5 Done Sent Done false
146072024-02-06T22:21:50.326Z 22 Acked 1021 Flush 0 Done Sent Done false
146082024-02-06T22:21:50.326Z 23 Acked 1022 Write 5 Done Sent Done false
146092024-02-06T22:21:50.326Z 24 Acked 1023 Read 4 Done Sent Done false
146102024-02-06T22:21:50.326Z 25 Acked 1024 Read 3 Done Sent Done false
146112024-02-06T22:21:50.326Z 26 Acked 1025 Write 1 Done Sent Done false
146122024-02-06T22:21:50.326Z 27 Acked 1026 Read 7 Done Sent Done false
146132024-02-06T22:21:50.326Z 28 Acked 1027 Read 10 Done Sent Done false
146142024-02-06T22:21:50.326Z 29 Acked 1028 Write 8 Done Sent Done false
146152024-02-06T22:21:50.326Z 30 Acked 1029 Read 2 Sent Sent Done false
146162024-02-06T22:21:50.326Z STATES DS:0 DS:1 DS:2 TOTAL
146172024-02-06T22:21:50.326Z New 0 0 0 0
146182024-02-06T22:21:50.326Z Sent 1 29 0 30
146192024-02-06T22:21:50.326Z Done 28 0 29 57
146202024-02-06T22:21:50.326Z Skipped 0 0 0 0
146212024-02-06T22:21:50.326Z Error 0 0 0 0
146222024-02-06T22:21:50.327ZLast Flush: 1021 1000 1021
146232024-02-06T22:21:50.327ZDownstairs last five completed: 1000
146242024-02-06T22:21:50.327ZUpstairs last five completed: 30 29 28 27 26
146252024-02-06T22:21:50.327Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:49.514739543Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1674}
146262024-02-06T22:21:50.327ZWrote out file "/tmp/test_repair_verify.data"
146272024-02-06T22:21:50.333Z
146282024-02-06T22:21:50.336ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146292024-02-06T22:21:50.346ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146302024-02-06T22:21:50.346Z 0 000-019 130 130 130 246 246 246 F F F
146312024-02-06T22:21:50.346Z 1 020-039 130 130 130 246 246 246 F F F
146322024-02-06T22:21:50.346Z 2 040-059 128 128 128 243 243 243 F F F
146332024-02-06T22:21:50.346Z 3 060-079 133 133 133 249 249 249 F F F
146342024-02-06T22:21:50.346Z 4 080-099 134 133 134 251 249 251 F F F <---
146352024-02-06T22:21:50.346Z 5 100-119 133 133 133 249 249 249 F F F
146362024-02-06T22:21:50.347Z 6 120-139 133 133 133 249 249 249 F F F
146372024-02-06T22:21:50.347Z 7 140-159 133 133 133 249 249 249 F F F
146382024-02-06T22:21:50.347Z 8 160-179 128 128 128 243 243 243 F F F
146392024-02-06T22:21:50.347Z 9 180-199 128 128 128 242 242 242 F F F
146402024-02-06T22:21:50.350Z 10 200-219 134 132 134 252 248 252 F F F <---
146412024-02-06T22:21:50.350Z 11 220-239 133 133 133 249 249 249 F F F
146422024-02-06T22:21:50.350Z 12 240-259 134 126 134 252 239 252 F F F <---
146432024-02-06T22:21:50.350Z 13 260-279 134 129 134 252 244 252 F F F <---
146442024-02-06T22:21:50.350Z 14 280-299 129 129 129 244 244 244 F F F
146452024-02-06T22:21:50.350Z 15 300-319 132 132 132 248 248 248 F F F
146462024-02-06T22:21:50.350Z 16 320-339 132 132 132 248 248 248 F F F
146472024-02-06T22:21:50.350Z 17 340-359 134 133 134 252 249 252 F F F <---
146482024-02-06T22:21:50.350Z 18 360-379 131 131 131 247 247 247 T F T <---
146492024-02-06T22:21:50.351Z 19 380-399 118 118 118 222 222 222 F F F
146502024-02-06T22:21:50.351Z 20 400-419 130 130 130 246 246 246 F F F
146512024-02-06T22:21:50.351Z 21 420-439 130 130 130 246 246 246 F F F
146522024-02-06T22:21:50.351Z 22 440-459 131 131 131 247 247 247 F F F
146532024-02-06T22:21:50.351Z 23 460-479 122 122 122 230 230 230 F F F
146542024-02-06T22:21:50.351Z 24 480-499 126 126 126 240 240 240 F F F
146552024-02-06T22:21:50.351Z 25 500-519 125 125 125 236 236 236 F F F
146562024-02-06T22:21:50.351Z 26 520-539 131 131 131 247 247 247 F F F
146572024-02-06T22:21:50.351Z 27 540-559 131 131 131 247 247 247 F F F
146582024-02-06T22:21:50.351Z 28 560-579 126 126 126 240 240 240 T F T <---
146592024-02-06T22:21:50.351Z 29 580-599 127 127 127 241 241 241 F F F
146602024-02-06T22:21:50.351ZMax gen: 134, Max flush: 252
146612024-02-06T22:21:50.351ZError: Difference in extent metadata found!
146622024-02-06T22:21:50.351ZA Difference in extent metadata is expected here
146632024-02-06T22:21:50.351ZOn loop 67
146642024-02-06T22:21:50.351Z
146652024-02-06T22:21:50.355ZVerifying data now
146662024-02-06T22:21:54.742ZNo change after verify
146672024-02-06T22:21:54.742ZLoop: 67 Downstairs dump after verify (and repair):
146682024-02-06T22:21:54.758ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146692024-02-06T22:21:54.758Z 0 000-019 130 130 130 246 246 246 F F F
146702024-02-06T22:21:54.758Z 1 020-039 130 130 130 246 246 246 F F F
146712024-02-06T22:21:54.758Z 2 040-059 128 128 128 243 243 243 F F F
146722024-02-06T22:21:54.758Z 3 060-079 133 133 133 249 249 249 F F F
146732024-02-06T22:21:54.758Z 4 080-099 134 134 134 251 251 251 F F F
146742024-02-06T22:21:54.758Z 5 100-119 133 133 133 249 249 249 F F F
146752024-02-06T22:21:54.758Z 6 120-139 133 133 133 249 249 249 F F F
146762024-02-06T22:21:54.758Z 7 140-159 133 133 133 249 249 249 F F F
146772024-02-06T22:21:54.758Z 8 160-179 128 128 128 243 243 243 F F F
146782024-02-06T22:21:54.758Z 9 180-199 128 128 128 242 242 242 F F F
146792024-02-06T22:21:54.761Z 10 200-219 134 134 134 252 252 252 F F F
146802024-02-06T22:21:54.762Z 11 220-239 133 133 133 249 249 249 F F F
146812024-02-06T22:21:54.762Z 12 240-259 134 134 134 252 252 252 F F F
146822024-02-06T22:21:54.762Z 13 260-279 134 134 134 252 252 252 F F F
146832024-02-06T22:21:54.762Z 14 280-299 129 129 129 244 244 244 F F F
146842024-02-06T22:21:54.762Z 15 300-319 132 132 132 248 248 248 F F F
146852024-02-06T22:21:54.762Z 16 320-339 132 132 132 248 248 248 F F F
146862024-02-06T22:21:54.762Z 17 340-359 134 134 134 252 252 252 F F F
146872024-02-06T22:21:54.762Z 18 360-379 135 135 135 253 253 253 F F F
146882024-02-06T22:21:54.762Z 19 380-399 118 118 118 222 222 222 F F F
146892024-02-06T22:21:54.762Z 20 400-419 130 130 130 246 246 246 F F F
146902024-02-06T22:21:54.762Z 21 420-439 130 130 130 246 246 246 F F F
146912024-02-06T22:21:54.762Z 22 440-459 131 131 131 247 247 247 F F F
146922024-02-06T22:21:54.762Z 23 460-479 122 122 122 230 230 230 F F F
146932024-02-06T22:21:54.762Z 24 480-499 126 126 126 240 240 240 F F F
146942024-02-06T22:21:54.762Z 25 500-519 125 125 125 236 236 236 F F F
146952024-02-06T22:21:54.762Z 26 520-539 131 131 131 247 247 247 F F F
146962024-02-06T22:21:54.762Z 27 540-559 131 131 131 247 247 247 F F F
146972024-02-06T22:21:54.762Z 28 560-579 135 135 135 253 253 253 F F F
146982024-02-06T22:21:54.762Z 29 580-599 127 127 127 241 241 241 F F F
146992024-02-06T22:21:54.762ZMax gen: 135, Max flush: 253
147002024-02-06T22:21:54.762Z
147012024-02-06T22:21:54.762ZBegin loop 68
147022024-02-06T22:21:54.762ZDownstairs to restart: 1
147032024-02-06T22:21:54.779Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.969558931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682}
147042024-02-06T22:21:54.779Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.969816366Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682}
147052024-02-06T22:21:54.785Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.969845602Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682}
147062024-02-06T22:21:54.785ZCrucible runtime is spawned
147072024-02-06T22:21:54.786Z{"msg":"Crucible e849514c-a951-446f-80be-7dfa08d31fa2 has session id: 14115752-f472-4502-9801-b69ecf05a1ce","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.969918503Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147082024-02-06T22:21:54.786Z{"msg":"Upstairs opts: Upstairs UUID: e849514c-a951-446f-80be-7dfa08d31fa2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.969970386Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147092024-02-06T22:21:54.786Z{"msg":"Crucible stats registered with UUID: e849514c-a951-446f-80be-7dfa08d31fa2","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970004991Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147102024-02-06T22:21:54.786Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970022529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682}
147112024-02-06T22:21:54.786Z{"msg":"e849514c-a951-446f-80be-7dfa08d31fa2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970130214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147122024-02-06T22:21:54.786Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970186967Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147132024-02-06T22:21:54.786Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970206534Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147142024-02-06T22:21:54.786Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.97023444Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147152024-02-06T22:21:54.786Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970270665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147162024-02-06T22:21:54.786Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970291882Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147172024-02-06T22:21:54.786Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970320868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147182024-02-06T22:21:54.786Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970338226Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147192024-02-06T22:21:54.786Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970368672Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147202024-02-06T22:21:54.786Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970401447Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147212024-02-06T22:21:54.786Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:53.97053275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147222024-02-06T22:21:54.786Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:53.970588432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147232024-02-06T22:21:54.786Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970632396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147242024-02-06T22:21:54.786Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970661462Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147252024-02-06T22:21:54.786Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:53.970701307Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147262024-02-06T22:21:54.786Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970715865Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147272024-02-06T22:21:54.786Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970746501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147282024-02-06T22:21:54.786Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.970923167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147292024-02-06T22:21:54.786Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.971374446Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147302024-02-06T22:21:54.787Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.971424629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147312024-02-06T22:21:54.787Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.971454955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147322024-02-06T22:21:54.787Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.9714916Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147332024-02-06T22:21:54.787Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.971994892Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147342024-02-06T22:21:54.787Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.972140972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147352024-02-06T22:21:54.787Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.972544168Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147362024-02-06T22:21:54.787Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.972587622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147372024-02-06T22:21:54.787Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.97260589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147382024-02-06T22:21:54.787Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.972628926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147392024-02-06T22:21:54.787Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.972644055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147402024-02-06T22:21:54.787Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:53.972665112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147412024-02-06T22:21:55.781Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.97117959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147422024-02-06T22:21:55.786Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.971419428Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147432024-02-06T22:21:55.786Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.972102536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147442024-02-06T22:21:55.786Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.972170236Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147452024-02-06T22:21:55.786Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.972603338Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147462024-02-06T22:21:55.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973072585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147472024-02-06T22:21:55.786Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973129847Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147482024-02-06T22:21:55.786Z{"msg":"[0]R flush_numbers[0..12]: [246, 246, 243, 249, 251, 249, 249, 249, 243, 242, 252, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973165332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147492024-02-06T22:21:55.786Z{"msg":"[0]R generation[0..12]: [130, 130, 128, 133, 134, 133, 133, 133, 128, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973191829Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147502024-02-06T22:21:55.787Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973224774Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147512024-02-06T22:21:55.787Z{"msg":"[1]R flush_numbers[0..12]: [246, 246, 243, 249, 251, 249, 249, 249, 243, 242, 252, 249]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
147522024-02-06T22:21:55.787Z,"time":"2024-02-06T22:21:54.97325137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147532024-02-06T22:21:55.787Z{"msg":"[1]R generation[0..12]: [130, 130, 128, 133, 134, 133, 133, 133, 128, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973276687Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147542024-02-06T22:21:55.787Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973309962Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147552024-02-06T22:21:55.787Z{"msg":"[2]R flush_numbers[0..12]: [246, 246, 243, 249, 251, 249, 249, 249, 243, 242, 252, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973336189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147562024-02-06T22:21:55.787Z{"msg":"[2]R generation[0..12]: [130, 130, 128, 133, 134, 133, 133, 133, 128, 128, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973366385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147572024-02-06T22:21:55.787Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973392371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147582024-02-06T22:21:55.787Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973421098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147592024-02-06T22:21:55.787Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973445804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147602024-02-06T22:21:55.787Z{"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973472171Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147612024-02-06T22:21:55.787Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973495437Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147622024-02-06T22:21:55.787Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973523034Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147632024-02-06T22:21:55.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973544971Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147642024-02-06T22:21:55.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973572637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
147652024-02-06T22:21:55.787Z14115752-f472-4502-9801-b69ecf05a1ce"}
147662024-02-06T22:21:55.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973600143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147672024-02-06T22:21:55.787Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973630529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147682024-02-06T22:21:55.787Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973653276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147692024-02-06T22:21:55.787Z{"msg":"e849514c-a951-446f-80be-7dfa08d31fa2 is now active with session: 14115752-f472-4502-9801-b69ecf05a1ce","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973680503Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147702024-02-06T22:21:55.787Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973701869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682,"session_id":"14115752-f472-4502-9801-b69ecf05a1ce"}
147712024-02-06T22:21:55.787Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:21:54.973727436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682}
147722024-02-06T22:21:55.787ZLoading write count information from file "/tmp/test_repair_verify.data"
147732024-02-06T22:21:55.787ZRun Repair workload
147742024-02-06T22:21:55.788Z01/30 Write block 553 len 12288 data: 10 9 8
147752024-02-06T22:21:55.791Z02/30 Flush
147762024-02-06T22:21:55.791Z03/30 Write block 509 len 28672 data: 12 11 10 9 13 15 13
147772024-02-06T22:21:55.791Z04/30 Write block 48 len 24576 data: 10 13 14 17 17 20
147782024-02-06T22:21:55.791Z05/30 Read block 378 len 36864
147792024-02-06T22:21:55.791Z06/30 Read block 335 len 32768
147802024-02-06T22:21:55.791Z07/30 Read block 358 len 8192
147812024-02-06T22:21:55.794Z08/30 Read block 410 len 12288
147822024-02-06T22:21:55.794Z09/30 Read block 129 len 16384
147832024-02-06T22:21:55.794Z10/30 Write block 59 len 20480 data: 11 11 9 7 7
147842024-02-06T22:21:55.794Z11/30 Read block 429 len 12288
147852024-02-06T22:21:55.797Z12/30 Write block 529 len 24576 data: 10 14 15 13 16 13
147862024-02-06T22:21:55.797Z13/30 Write block 467 len 8192 data: 12 12
147872024-02-06T22:21:55.797Z14/30 Write block 527 len 12288 data: 8 8 11
147882024-02-06T22:21:55.797Z15/30 Read block 507 len 24576
147892024-02-06T22:21:55.800Z16/30 Write block 155 len 24576 data: 17 14 16 12 13 11
147902024-02-06T22:21:55.800Z17/30 Flush
147912024-02-06T22:21:55.803Z18/30 Write block 288 len 12288 data: 10 11 15
147922024-02-06T22:21:55.803Z19/30 Write block 356 len 24576 data: 17 18 16 14 13 12
147932024-02-06T22:21:55.803Z20/30 Read block 180 len 4096
147942024-02-06T22:21:55.806Z21/30 Write block 166 len 28672 data: 9 11 14 15 15 12 13
147952024-02-06T22:21:55.806Z22/30 Read block 517 len 16384
147962024-02-06T22:21:55.806Z23/30 Read block 343 len 12288
147972024-02-06T22:21:55.809Z24/30 Read block 421 len 24576
147982024-02-06T22:21:55.809Z25/30 Read block 213 len 12288
147992024-02-06T22:21:55.809Z26/30 Read block 257 len 40960
148002024-02-06T22:21:55.809Z27/30 Read block 474 len 40960
148012024-02-06T22:21:55.812Z28/30 Write block 181 len 12288 data: 16 14 18
148022024-02-06T22:21:55.812Z29/30 Read block 34 len 32768
148032024-02-06T22:21:55.812Z30/30 Read block 204 len 12288
148042024-02-06T22:21:55.815Z----------------------------------------------------------------
148052024-02-06T22:21:55.815Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:28
148062024-02-06T22:21:55.815ZStates: Active Active Active
148072024-02-06T22:21:55.815ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148082024-02-06T22:21:55.815Z 3 Acked 1002 Write 7 Done Done Done false
148092024-02-06T22:21:55.815Z 4 Acked 1003 Write 6 Done Sent Done false
148102024-02-06T22:21:55.815Z 5 Acked 1004 Read 9 Done Sent Done false
148112024-02-06T22:21:55.815Z 6 Acked 1005 Read 8 Done Sent Done false
148122024-02-06T22:21:55.815Z 7 Acked 1006 Read 2 Done Sent Done false
148132024-02-06T22:21:55.815Z 8 Acked 1007 Read 3 Done Sent Done false
148142024-02-06T22:21:55.815Z 9 Acked 1008 Read 4 Done Sent Done false
148152024-02-06T22:21:55.816Z 10 Acked 1009 Write 5 Done Sent Done false
148162024-02-06T22:21:55.816Z 11 Acked 1010 Read 3 Done Sent Done false
148172024-02-06T22:21:55.816Z 12 Acked 1011 Write 6 Done Sent Done false
148182024-02-06T22:21:55.816Z 13 Acked 1012 Write 2 Done Sent Done false
148192024-02-06T22:21:55.816Z 14 Acked 1013 Write 3 Done Sent Done false
148202024-02-06T22:21:55.816Z 15 Acked 1014 Read 6 Done Sent Done false
148212024-02-06T22:21:55.816Z 16 Acked 1015 Write 6 Done Sent Done false
148222024-02-06T22:21:55.816Z 17 Acked 1016 Flush 0 Done Sent Done false
148232024-02-06T22:21:55.816Z 18 Acked 1017 Write 3 Done Sent Done false
148242024-02-06T22:21:55.816Z 19 Acked 1018 Write 6 Done Sent Done false
148252024-02-06T22:21:55.816Z 20 Acked 1019 Read 1 Done Sent Done false
148262024-02-06T22:21:55.816Z 21 Acked 1020 Write 7 Done Sent Done false
148272024-02-06T22:21:55.816Z 22 Acked 1021 Read 4 Done Sent Done false
148282024-02-06T22:21:55.816Z 23 Acked 1022 Read 3 Done Sent Done false
148292024-02-06T22:21:55.816Z 24 Acked 1023 Read 6 Done Sent Done false
148302024-02-06T22:21:55.816Z 25 Acked 1024 Read 3 Done Sent Done false
148312024-02-06T22:21:55.816Z 26 Acked 1025 Read 10 Done Sent Done false
148322024-02-06T22:21:55.816Z 27 Acked 1026 Read 10 Done Sent Done false
148332024-02-06T22:21:55.816Z 28 Acked 1027 Write 3 Done Sent Done false
148342024-02-06T22:21:55.816Z 29 Acked 1028 Read 8 Done Sent Done false
148352024-02-06T22:21:55.816Z 30 Acked 1029 Read 3 Sent Sent Done false
148362024-02-06T22:21:55.816Z STATES DS:0 DS:1 DS:2 TOTAL
148372024-02-06T22:21:55.816Z New 0 0 0 0
148382024-02-06T22:21:55.816Z Sent 1 27 0 28
148392024-02-06T22:21:55.817Z Done 27 1 28 56
148402024-02-06T22:21:55.817Z Skipped 0 0 0 0
148412024-02-06T22:21:55.817Z Error 0 0 0 0
148422024-02-06T22:21:55.817ZLast Flush: 1016 1001 1016
148432024-02-06T22:21:55.817ZDownstairs last five completed: 1001 1000
148442024-02-06T22:21:55.817ZUpstairs last five completed: 30 29 28 27 26
148452024-02-06T22:21:55.817Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:21:55.002868319Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1682}
148462024-02-06T22:21:55.817ZWrote out file "/tmp/test_repair_verify.data"
148472024-02-06T22:21:55.820Z
148482024-02-06T22:21:55.820ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
148492024-02-06T22:21:55.838ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148502024-02-06T22:21:55.838Z 0 000-019 130 130 130 246 246 246 F F F
148512024-02-06T22:21:55.838Z 1 020-039 130 130 130 246 246 246 F F F
148522024-02-06T22:21:55.838Z 2 040-059 136 128 136 255 243 255 F F F <---
148532024-02-06T22:21:55.841Z 3 060-079 136 133 136 255 249 255 F F F <---
148542024-02-06T22:21:55.841Z 4 080-099 134 134 134 251 251 251 F F F
148552024-02-06T22:21:55.841Z 5 100-119 133 133 133 249 249 249 F F F
148562024-02-06T22:21:55.841Z 6 120-139 133 133 133 249 249 249 F F F
148572024-02-06T22:21:55.841Z 7 140-159 136 133 136 255 249 255 F F F <---
148582024-02-06T22:21:55.841Z 8 160-179 136 128 136 255 243 255 T F T <---
148592024-02-06T22:21:55.841Z 9 180-199 128 128 128 242 242 242 T F T <---
148602024-02-06T22:21:55.841Z 10 200-219 134 134 134 252 252 252 F F F
148612024-02-06T22:21:55.841Z 11 220-239 133 133 133 249 249 249 F F F
148622024-02-06T22:21:55.841Z 12 240-259 134 134 134 252 252 252 F F F
148632024-02-06T22:21:55.842Z 13 260-279 134 134 134 252 252 252 F F F
148642024-02-06T22:21:55.842Z 14 280-299 129 129 129 244 244 244 T F T <---
148652024-02-06T22:21:55.842Z 15 300-319 132 132 132 248 248 248 F F F
148662024-02-06T22:21:55.842Z 16 320-339 132 132 132 248 248 248 F F F
148672024-02-06T22:21:55.842Z 17 340-359 134 134 134 252 252 252 T F T <---
148682024-02-06T22:21:55.842Z 18 360-379 135 135 135 253 253 253 T F T <---
148692024-02-06T22:21:55.842Z 19 380-399 118 118 118 222 222 222 F F F
148702024-02-06T22:21:55.842Z 20 400-419 130 130 130 246 246 246 F F F
148712024-02-06T22:21:55.842Z 21 420-439 130 130 130 246 246 246 F F F
148722024-02-06T22:21:55.842Z 22 440-459 131 131 131 247 247 247 F F F
148732024-02-06T22:21:55.842Z 23 460-479 136 122 136 255 230 255 F F F <---
148742024-02-06T22:21:55.842Z 24 480-499 126 126 126 240 240 240 F F F
148752024-02-06T22:21:55.842Z 25 500-519 136 125 136 255 236 255 F T F <---
148762024-02-06T22:21:55.842Z 26 520-539 136 131 136 255 247 255 F F F <---
148772024-02-06T22:21:55.842Z 27 540-559 136 136 136 254 254 254 F F F
148782024-02-06T22:21:55.842Z 28 560-579 135 135 135 253 253 253 F F F
148792024-02-06T22:21:55.842Z 29 580-599 127 127 127 241 241 241 F F F
148802024-02-06T22:21:55.842ZMax gen: 136, Max flush: 255
148812024-02-06T22:21:55.842ZError: Difference in extent metadata found!
148822024-02-06T22:21:55.842ZA Difference in extent metadata is expected here
148832024-02-06T22:21:55.842ZOn loop 68
148842024-02-06T22:21:55.845Z
148852024-02-06T22:21:55.846ZVerifying data now
148862024-02-06T22:22:01.414ZNo change after verify
148872024-02-06T22:22:01.414ZLoop: 68 Downstairs dump after verify (and repair):
148882024-02-06T22:22:01.434ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148892024-02-06T22:22:01.434Z 0 000-019 130 130 130 246 246 246 F F F
148902024-02-06T22:22:01.435Z 1 020-039 130 130 130 246 246 246 F F F
148912024-02-06T22:22:01.435Z 2 040-059 136 136 136 255 255 255 F F F
148922024-02-06T22:22:01.435Z 3 060-079 136 136 136 255 255 255 F F F
148932024-02-06T22:22:01.435Z 4 080-099 134 134 134 251 251 251 F F F
148942024-02-06T22:22:01.435Z 5 100-119 133 133 133 249 249 249 F F F
148952024-02-06T22:22:01.435Z 6 120-139 133 133 133 249 249 249 F F F
148962024-02-06T22:22:01.435Z 7 140-159 136 136 136 255 255 255 F F F
148972024-02-06T22:22:01.435Z 8 160-179 137 137 137 256 256 256 F F F
148982024-02-06T22:22:01.435Z 9 180-199 137 137 137 256 256 256 F F F
148992024-02-06T22:22:01.435Z 10 200-219 134 134 134 252 252 252 F F F
149002024-02-06T22:22:01.436Z 11 220-239 133 133 133 249 249 249 F F F
149012024-02-06T22:22:01.436Z 12 240-259 134 134 134 252 252 252 F F F
149022024-02-06T22:22:01.436Z 13 260-279 134 134 134 252 252 252 F F F
149032024-02-06T22:22:01.436Z 14 280-299 137 137 137 256 256 256 F F F
149042024-02-06T22:22:01.436Z 15 300-319 132 132 132 248 248 248 F F F
149052024-02-06T22:22:01.436Z 16 320-339 132 132 132 248 248 248 F F F
149062024-02-06T22:22:01.436Z 17 340-359 137 137 137 256 256 256 F F F
149072024-02-06T22:22:01.436Z 18 360-379 137 137 137 256 256 256 F F F
149082024-02-06T22:22:01.436Z 19 380-399 118 118 118 222 222 222 F F F
149092024-02-06T22:22:01.436Z 20 400-419 130 130 130 246 246 246 F F F
149102024-02-06T22:22:01.437Z 21 420-439 130 130 130 246 246 246 F F F
149112024-02-06T22:22:01.437Z 22 440-459 131 131 131 247 247 247 F F F
149122024-02-06T22:22:01.437Z 23 460-479 136 136 136 255 255 255 F F F
149132024-02-06T22:22:01.437Z 24 480-499 126 126 126 240 240 240 F F F
149142024-02-06T22:22:01.437Z 25 500-519 136 136 136 255 255 255 F F F
149152024-02-06T22:22:01.437Z 26 520-539 136 136 136 255 255 255 F F F
149162024-02-06T22:22:01.437Z 27 540-559 136 136 136 254 254 254 F F F
149172024-02-06T22:22:01.438Z 28 560-579 135 135 135 253 253 253 F F F
149182024-02-06T22:22:01.438Z 29 580-599 127 127 127 241 241 241 F F F
149192024-02-06T22:22:01.438ZMax gen: 137, Max flush: 256
149202024-02-06T22:22:01.438Z
149212024-02-06T22:22:01.438ZBegin loop 69
149222024-02-06T22:22:01.438ZDownstairs to restart: 1
149232024-02-06T22:22:01.883Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627045234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690}
149242024-02-06T22:22:01.883Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible"Crucible runtime is spawned
149252024-02-06T22:22:01.883Z,"level":30,"time":"2024-02-06T22:22:00.627345604Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690}
149262024-02-06T22:22:01.884Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627390047Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690}
149272024-02-06T22:22:01.884Z{"msg":"Crucible 6247b3e0-4a3f-4db4-a3aa-f58c10a4f62b has session id: bb533af0-074e-4389-8a49-c9c5b202bf9c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627419103Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149282024-02-06T22:22:01.884Z{"msg":"Upstairs opts: Upstairs UUID: 6247b3e0-4a3f-4db4-a3aa-f58c10a4f62b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627475666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149292024-02-06T22:22:01.884Z{"msg":"Crucible stats registered with UUID: 6247b3e0-4a3f-4db4-a3aa-f58c10a4f62b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.62751844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149302024-02-06T22:22:01.884Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627563844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690}
149312024-02-06T22:22:01.884Z{"msg":"6247b3e0-4a3f-4db4-a3aa-f58c10a4f62b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.62759215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149322024-02-06T22:22:01.884Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627624746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149332024-02-06T22:22:01.884Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627657181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149342024-02-06T22:22:01.884Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.62773794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149352024-02-06T22:22:01.884Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627765517Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149362024-02-06T22:22:01.884Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627790653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149372024-02-06T22:22:01.884Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.62781639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149382024-02-06T22:22:01.885Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627863934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149392024-02-06T22:22:01.885Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.62789234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149402024-02-06T22:22:01.885Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.627939403Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149412024-02-06T22:22:01.885Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:00.627973849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149422024-02-06T22:22:01.885Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:00.62803466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149432024-02-06T22:22:01.885Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.628074225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149442024-02-06T22:22:01.885Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:00.628105211Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149452024-02-06T22:22:01.885Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.628138276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149462024-02-06T22:22:01.886Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.628161253Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149472024-02-06T22:22:01.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.62818477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149482024-02-06T22:22:01.886Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.628210377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149492024-02-06T22:22:01.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.62848529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149502024-02-06T22:22:01.886Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.628522634Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149512024-02-06T22:22:01.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.628539182Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149522024-02-06T22:22:01.886Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.628570578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149532024-02-06T22:22:01.886Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629126053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149542024-02-06T22:22:01.887Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629171357Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149552024-02-06T22:22:01.887Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629699326Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149562024-02-06T22:22:01.887Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629735031Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149572024-02-06T22:22:01.887Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629750608Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149582024-02-06T22:22:01.887Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629765306Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149592024-02-06T22:22:01.887Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629787534Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149602024-02-06T22:22:01.887Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:00.629801722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149612024-02-06T22:22:02.441Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.629516598Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149622024-02-06T22:22:02.441Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.629649Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149632024-02-06T22:22:02.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.630518543Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149642024-02-06T22:22:02.448Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.630563317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149652024-02-06T22:22:02.448Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631070398Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149662024-02-06T22:22:02.448Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631593638Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149672024-02-06T22:22:02.448Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.63164472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149682024-02-06T22:22:02.448Z{"msg":"[0]R flush_numbers[0..12]: [246, 246, 255, 255, 251, 249, 249, 255, 256, 256, 252, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631681815Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149692024-02-06T22:22:02.448Z{"msg":"[0]R generation[0..12]: [130, 130, 136, 136, 134, 133, 133, 136, 137, 137, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631726869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149702024-02-06T22:22:02.448Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631758705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149712024-02-06T22:22:02.448Z{"msg":"[1]R flush_numbers[0..12]: [246, 246, 255, 255, 251, 249, 249, 255, 256, 256, 252, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.63179549Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149722024-02-06T22:22:02.448Z{"msg":"Wait for a query_work_queue command to finish before sending IO
149732024-02-06T22:22:02.448Z[1]R generation[0..12]: [130, 130, 136, 136, 134, 133, 133, 136, 137, 137, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631821227Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149742024-02-06T22:22:02.448Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631861281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149752024-02-06T22:22:02.448Z{"msg":"[2]R flush_numbers[0..12]: [246, 246, 255, 255, 251, 249, 249, 255, 256, 256, 252, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631887468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149762024-02-06T22:22:02.448Z{"msg":"[2]R generation[0..12]: [130, 130, 136, 136, 134, 133, 133, 136, 137, 137, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631923383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149772024-02-06T22:22:02.448Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.63194536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149782024-02-06T22:22:02.449Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.631982015Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149792024-02-06T22:22:02.449Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632001632Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149802024-02-06T22:22:02.449Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.63202244Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149812024-02-06T22:22:02.449Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632059424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149822024-02-06T22:22:02.449Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632080922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149832024-02-06T22:22:02.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632116287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149842024-02-06T22:22:02.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632136294Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149852024-02-06T22:22:02.449Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632172909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149862024-02-06T22:22:02.449Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
149872024-02-06T22:22:02.449ZAll required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632198686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149882024-02-06T22:22:02.449Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632233091Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149892024-02-06T22:22:02.449Z{"msg":"6247b3e0-4a3f-4db4-a3aa-f58c10a4f62b is now active with session: bb533af0-074e-4389-8a49-c9c5b202bf9c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632256678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149902024-02-06T22:22:02.449Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632290853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690,"session_id":"bb533af0-074e-4389-8a49-c9c5b202bf9c"}
149912024-02-06T22:22:02.449Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:01.632327488Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690}
149922024-02-06T22:22:02.449ZLoading write count information from file "/tmp/test_repair_verify.data"
149932024-02-06T22:22:02.449ZRun Repair workload
149942024-02-06T22:22:02.449Z01/30 Write block 508 len 36864 data: 13 13 12 11 10 14 16 14 11
149952024-02-06T22:22:02.453Z02/30 Read block 122 len 28672
149962024-02-06T22:22:02.453Z03/30 Read block 133 len 8192
149972024-02-06T22:22:02.453Z04/30 Read block 407 len 8192
149982024-02-06T22:22:02.453Z05/30 Read block 388 len 20480
149992024-02-06T22:22:02.453Z06/30 Write block 132 len 8192 data: 12 13
150002024-02-06T22:22:02.453Z07/30 Flush
150012024-02-06T22:22:02.453Z08/30 Write block 162 len 16384 data: 13 11 11 9
150022024-02-06T22:22:02.453Z09/30 Write block 374 len 8192 data: 10 9
150032024-02-06T22:22:02.453Z10/30 Read block 50 len 32768
150042024-02-06T22:22:02.456Z11/30 Write block 262 len 28672 data: 17 18 16 17 18 18 16
150052024-02-06T22:22:02.456Z12/30 Write block 22 len 32768 data: 9 9 8 9 8 8 9 7
150062024-02-06T22:22:02.456Z13/30 Flush
150072024-02-06T22:22:02.460Z14/30 Write block 123 len 36864 data: 13 13 14 12 14 15 14 15 14
150082024-02-06T22:22:02.460Z15/30 Read block 209 len 24576
150092024-02-06T22:22:02.465Z16/30 Write block 521 len 36864 data: 7 7 6 6 6 9 9 9 12
150102024-02-06T22:22:02.465Z17/30 Write block 86 len 20480 data: 9 10 9 9 7
150112024-02-06T22:22:02.465Z18/30 Read block 543 len 8192
150122024-02-06T22:22:02.465Z19/30 Write block 51 len 40960 data: 18 18 21 22 20 19 15 13 12 12
150132024-02-06T22:22:02.465Z20/30 Read block 168 len 4096
150142024-02-06T22:22:02.468Z21/30 Write block 218 len 28672 data: 10 10 15 15 14 14 14
150152024-02-06T22:22:02.468Z22/30 Read block 468 len 20480
150162024-02-06T22:22:02.468Z23/30 Write block 580 len 4096 data: 11
150172024-02-06T22:22:02.468Z24/30 Read block 305 len 40960
150182024-02-06T22:22:02.473Z25/30 Flush
150192024-02-06T22:22:02.473Z26/30 Write block 241 len 24576 data: 7 8 7 7 9 9
150202024-02-06T22:22:02.476Z27/30 Write block 439 len 20480 data: 12 11 8 10 8
150212024-02-06T22:22:02.477Z28/30 Write block 429 len 12288 data: 10 9 5
150222024-02-06T22:22:02.477Z29/30 Read block 307 len 24576
150232024-02-06T22:22:02.477Z30/30 Write block 76 len 32768 data: 10 11 10 7 7 7 10 10
150242024-02-06T22:22:02.477Z----------------------------------------------------------------
150252024-02-06T22:22:02.477Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:23
150262024-02-06T22:22:02.477ZStates: Active Active Active
150272024-02-06T22:22:02.477ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
150282024-02-06T22:22:02.477Z 8 Acked 1007 Write 4 Done Done Done false
150292024-02-06T22:22:02.477Z 9 Acked 1008 Write 2 Done Sent Done false
150302024-02-06T22:22:02.477Z 10 Acked 1009 Read 8 Done Sent Done false
150312024-02-06T22:22:02.477Z 11 Acked 1010 Write 7 Done Sent Done false
150322024-02-06T22:22:02.477Z 12 Acked 1011 Write 8 Done Sent Done false
150332024-02-06T22:22:02.477Z 13 Acked 1012 Flush 0 Done Sent Done false
150342024-02-06T22:22:02.477Z 14 Acked 1013 Write 9 Done Sent Done false
150352024-02-06T22:22:02.477Z 15 Acked 1014 Read 6 Done Sent Done false
150362024-02-06T22:22:02.477Z 16 Acked 1015 Write 9 Done Sent Done false
150372024-02-06T22:22:02.477Z 17 Acked 1016 Write 5 Done Sent Done false
150382024-02-06T22:22:02.477Z 18 Acked 1017 Read 2 Done Sent Done false
150392024-02-06T22:22:02.477Z 19 Acked 1018 Write 10 Done Sent Done false
150402024-02-06T22:22:02.477Z 20 Acked 1019 Read 1 Done Sent Done false
150412024-02-06T22:22:02.477Z 21 Acked 1020 Write 7 Done Sent Done false
150422024-02-06T22:22:02.477Z 22 Acked 1021 Read 5 Done Sent Done false
150432024-02-06T22:22:02.477Z 23 Acked 1022 Write 1 Done Sent Done false
150442024-02-06T22:22:02.478Z 24 Acked 1023 Read 10 Done Sent Done false
150452024-02-06T22:22:02.478Z 25 Acked 1024 Flush 0 Done Sent Done false
150462024-02-06T22:22:02.478Z 26 Acked 1025 Write 6 Done Sent Done false
150472024-02-06T22:22:02.478Z 27 Acked 1026 Write 5 Done Sent Done false
150482024-02-06T22:22:02.478Z 28 Acked 1027 Write 3 Done Sent Done false
150492024-02-06T22:22:02.478Z 29 Acked 1028 Read 6 Done Sent Done false
150502024-02-06T22:22:02.478Z 30 Acked 1029 Write 8 Sent Sent Sent false
150512024-02-06T22:22:02.478Z STATES DS:0 DS:1 DS:2 TOTAL
150522024-02-06T22:22:02.478Z New 0 0 0 0
150532024-02-06T22:22:02.478Z Sent 1 22 1 24
150542024-02-06T22:22:02.478Z Done 22 1 22 45
150552024-02-06T22:22:02.478Z Skipped 0 0 0 0
150562024-02-06T22:22:02.478Z Error 0 0 0 0
150572024-02-06T22:22:02.478ZLast Flush: 1024 1006 1024
150582024-02-06T22:22:02.478ZDownstairs last five completed: 1006 1005 1004 1003 1002
150592024-02-06T22:22:02.478ZUpstairs last five completed: 30 29 28 27 26
150602024-02-06T22:22:02.478Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:01.664689386Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1690}
150612024-02-06T22:22:02.478ZWrote out file "/tmp/test_repair_verify.data"
150622024-02-06T22:22:02.482Z
150632024-02-06T22:22:02.485ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
150642024-02-06T22:22:02.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150652024-02-06T22:22:02.502Z 0 000-019 130 130 130 246 246 246 F F F
150662024-02-06T22:22:02.502Z 1 020-039 138 130 138 258 246 258 F F F <---
150672024-02-06T22:22:02.502Z 2 040-059 138 136 138 259 255 259 F F F <---
150682024-02-06T22:22:02.502Z 3 060-079 138 136 138 259 255 259 T F T <---
150692024-02-06T22:22:02.502Z 4 080-099 138 134 138 259 251 259 T F T <---
150702024-02-06T22:22:02.502Z 5 100-119 133 133 133 249 249 249 F F F
150712024-02-06T22:22:02.502Z 6 120-139 138 138 138 259 257 259 F F F <---
150722024-02-06T22:22:02.502Z 7 140-159 136 136 136 255 255 255 F F F
150732024-02-06T22:22:02.502Z 8 160-179 138 137 138 258 256 258 F T F <---
150742024-02-06T22:22:02.506Z 9 180-199 137 137 137 256 256 256 F F F
150752024-02-06T22:22:02.506Z 10 200-219 138 134 138 259 252 259 F F F <---
150762024-02-06T22:22:02.506Z 11 220-239 138 133 138 259 249 259 F F F <---
150772024-02-06T22:22:02.506Z 12 240-259 134 134 134 252 252 252 T F T <---
150782024-02-06T22:22:02.506Z 13 260-279 138 134 138 258 252 258 F F F <---
150792024-02-06T22:22:02.506Z 14 280-299 137 137 137 256 256 256 F F F
150802024-02-06T22:22:02.506Z 15 300-319 132 132 132 248 248 248 F F F
150812024-02-06T22:22:02.506Z 16 320-339 132 132 132 248 248 248 F F F
150822024-02-06T22:22:02.506Z 17 340-359 137 137 137 256 256 256 F F F
150832024-02-06T22:22:02.506Z 18 360-379 138 137 138 258 256 258 F F F <---
150842024-02-06T22:22:02.506Z 19 380-399 118 118 118 222 222 222 F F F
150852024-02-06T22:22:02.506Z 20 400-419 130 130 130 246 246 246 F F F
150862024-02-06T22:22:02.506Z 21 420-439 130 130 130 246 246 246 T F T <---
150872024-02-06T22:22:02.506Z 22 440-459 131 131 131 247 247 247 T F T <---
150882024-02-06T22:22:02.506Z 23 460-479 136 136 136 255 255 255 F F F
150892024-02-06T22:22:02.506Z 24 480-499 126 126 126 240 240 240 F F F
150902024-02-06T22:22:02.506Z 25 500-519 138 138 138 257 257 257 F F F
150912024-02-06T22:22:02.506Z 26 520-539 138 136 138 259 255 259 F F F <---
150922024-02-06T22:22:02.506Z 27 540-559 136 136 136 254 254 254 F F F
150932024-02-06T22:22:02.506Z 28 560-579 135 135 135 253 253 253 F F F
150942024-02-06T22:22:02.506Z 29 580-599 138 127 138 259 241 259 F F F <---
150952024-02-06T22:22:02.506ZMax gen: 138, Max flush: 259
150962024-02-06T22:22:02.507ZError: Difference in extent metadata found!
150972024-02-06T22:22:02.507ZA Difference in extent metadata is expected here
150982024-02-06T22:22:02.507ZOn loop 69
150992024-02-06T22:22:02.507Z
151002024-02-06T22:22:02.510ZVerifying data now
151012024-02-06T22:22:07.227ZNo change after verify
151022024-02-06T22:22:07.232ZLoop: 69 Downstairs dump after verify (and repair):
151032024-02-06T22:22:07.242ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151042024-02-06T22:22:07.243Z 0 000-019 130 130 130 246 246 246 F F F
151052024-02-06T22:22:07.243Z 1 020-039 138 138 138 258 258 258 F F F
151062024-02-06T22:22:07.243Z 2 040-059 138 138 138 259 259 259 F F F
151072024-02-06T22:22:07.243Z 3 060-079 139 139 139 260 260 260 F F F
151082024-02-06T22:22:07.243Z 4 080-099 139 139 139 260 260 260 F F F
151092024-02-06T22:22:07.243Z 5 100-119 133 133 133 249 249 249 F F F
151102024-02-06T22:22:07.244Z 6 120-139 138 138 138 259 259 259 F F F
151112024-02-06T22:22:07.244Z 7 140-159 136 136 136 255 255 255 F F F
151122024-02-06T22:22:07.244Z 8 160-179 138 138 138 258 258 258 F F F
151132024-02-06T22:22:07.244Z 9 180-199 137 137 137 256 256 256 F F F
151142024-02-06T22:22:07.244Z 10 200-219 138 138 138 259 259 259 F F F
151152024-02-06T22:22:07.245Z 11 220-239 138 138 138 259 259 259 F F F
151162024-02-06T22:22:07.257Z 12 240-259 139 139 139 260 260 260 F F F
151172024-02-06T22:22:07.257Z 13 260-279 138 138 138 258 258 258 F F F
151182024-02-06T22:22:07.257Z 14 280-299 137 137 137 256 256 256 F F F
151192024-02-06T22:22:07.257Z 15 300-319 132 132 132 248 248 248 F F F
151202024-02-06T22:22:07.258Z 16 320-339 132 132 132 248 248 248 F F F
151212024-02-06T22:22:07.258Z 17 340-359 137 137 137 256 256 256 F F F
151222024-02-06T22:22:07.258Z 18 360-379 138 138 138 258 258 258 F F F
151232024-02-06T22:22:07.258Z 19 380-399 118 118 118 222 222 222 F F F
151242024-02-06T22:22:07.259Z 20 400-419 130 130 130 246 246 246 F F F
151252024-02-06T22:22:07.259Z 21 420-439 139 139 139 260 260 260 F F F
151262024-02-06T22:22:07.259Z 22 440-459 139 139 139 260 260 260 F F F
151272024-02-06T22:22:07.260Z 23 460-479 136 136 136 255 255 255 F F F
151282024-02-06T22:22:07.260Z 24 480-499 126 126 126 240 240 240 F F F
151292024-02-06T22:22:07.260Z 25 500-519 138 138 138 257 257 257 F F F
151302024-02-06T22:22:07.260Z 26 520-539 138 138 138 259 259 259 F F F
151312024-02-06T22:22:07.260Z 27 540-559 136 136 136 254 254 254 F F F
151322024-02-06T22:22:07.261Z 28 560-579 135 135 135 253 253 253 F F F
151332024-02-06T22:22:07.261Z 29 580-599 138 138 138 259 259 259 F F F
151342024-02-06T22:22:07.261ZMax gen: 139, Max flush: 260
151352024-02-06T22:22:07.261Z
151362024-02-06T22:22:07.261ZBegin loop 70
151372024-02-06T22:22:07.261ZDownstairs to restart: 1
151382024-02-06T22:22:07.270Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451242145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698}
151392024-02-06T22:22:07.271Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
151402024-02-06T22:22:07.271Z,"time":"2024-02-06T22:22:06.451570161Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698}
151412024-02-06T22:22:07.271Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451608385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698}
151422024-02-06T22:22:07.271Z{"msg":"Crucible 6f8b5d16-39f5-49d3-abb1-eebd447b99c0 has session id: dce52edc-a5c8-4d1f-93e1-2fac29d5eec0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451626173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151432024-02-06T22:22:07.271Z{"msg":"Upstairs opts: Upstairs UUID: 6f8b5d16-39f5-49d3-abb1-eebd447b99c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451670037Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151442024-02-06T22:22:07.272Z{"msg":"Crucible stats registered with UUID: 6f8b5d16-39f5-49d3-abb1-eebd447b99c0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451715961Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151452024-02-06T22:22:07.272Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451746737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698}
151462024-02-06T22:22:07.272Z{"msg":"6f8b5d16-39f5-49d3-abb1-eebd447b99c0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451761985Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151472024-02-06T22:22:07.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451786241Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151482024-02-06T22:22:07.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451820867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151492024-02-06T22:22:07.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451853712Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151502024-02-06T22:22:07.273Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451909545Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151512024-02-06T22:22:07.273Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451954169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151522024-02-06T22:22:07.273Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.451981365Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151532024-02-06T22:22:07.273Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452006402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151542024-02-06T22:22:07.273Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452033368Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151552024-02-06T22:22:07.273Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452069513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151562024-02-06T22:22:07.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452097919Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151572024-02-06T22:22:07.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452123876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151582024-02-06T22:22:07.274Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:06.452153022Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151592024-02-06T22:22:07.274Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:06.452195876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151602024-02-06T22:22:07.274Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:06.452227372Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151612024-02-06T22:22:07.274Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452252458Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151622024-02-06T22:22:07.274Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452272706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151632024-02-06T22:22:07.275Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452454081Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151642024-02-06T22:22:07.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452816092Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151652024-02-06T22:22:07.275Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452864286Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151662024-02-06T22:22:07.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452891992Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151672024-02-06T22:22:07.275Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.452941735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151682024-02-06T22:22:07.275Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.453420561Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151692024-02-06T22:22:07.276Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.453565111Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151702024-02-06T22:22:07.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.453906235Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151712024-02-06T22:22:07.276Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.453957718Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151722024-02-06T22:22:07.276Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.45402036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151732024-02-06T22:22:07.276Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.454055485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151742024-02-06T22:22:07.276Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.454073292Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151752024-02-06T22:22:07.277Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:06.45409606Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151762024-02-06T22:22:08.274Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.453615282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151772024-02-06T22:22:08.303Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.45385211Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151782024-02-06T22:22:08.303Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.45451365Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151792024-02-06T22:22:08.303Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.454551375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151802024-02-06T22:22:08.303Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.454986976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151812024-02-06T22:22:08.303Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455468591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151822024-02-06T22:22:08.303Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455495908Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151832024-02-06T22:22:08.303Z{"msg":"[0]R flush_numbers[0..12]: [246, 258, 259, 260, 260, 249, 259, 255, 258, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455511985Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151842024-02-06T22:22:08.303Z{"msg":"[0]R generation[0..12]: [130, 138, 138, 139, 139, 133, 138, 136, 138, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455526634Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151852024-02-06T22:22:08.304Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.45555047Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151862024-02-06T22:22:08.304Z{"msg":"[1]R flush_numbers[0..12]: [246, 258, 259, 260, 260, 249, 259, 255, 258, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455565968Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151872024-02-06T22:22:08.304Z{"msg":"[1]R generation[0..12]: [130, 138, 138, 139, 139, 133, 138, 136, 138, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455587835Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151882024-02-06T22:22:08.304Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455602733Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151892024-02-06T22:22:08.304Z{"msg":"[2]R flush_numbers[0..12]: [246, 258, 259, 260, 260, 249, 259, 255, 258, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.45562509Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151902024-02-06T22:22:08.304Z{"msg":"[2]R generation[0..12]: [130, 138, 138, 139, 139, 133, 138, 136, 138, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455640658Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151912024-02-06T22:22:08.304Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455661735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151922024-02-06T22:22:08.304Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455675633Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151932024-02-06T22:22:08.304Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
151942024-02-06T22:22:08.304Z,"time":"2024-02-06T22:22:07.455689572Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151952024-02-06T22:22:08.304Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455706829Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151962024-02-06T22:22:08.304Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455720387Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151972024-02-06T22:22:08.304Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455734945Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151982024-02-06T22:22:08.304Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455747274Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
151992024-02-06T22:22:08.304Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.45577246Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
152002024-02-06T22:22:08.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455790518Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
152012024-02-06T22:22:08.305Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455805276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
152022024-02-06T22:22:08.305Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455827023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
152032024-02-06T22:22:08.305Z{"msg":"6f8b5d16-39f5-49d3-abb1-eebd447b99c0 is now active with session: dce52edc-a5c8-4d1f-93e1-2fac29d5eec0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455841721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
152042024-02-06T22:22:08.305Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455854009Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698,"session_id":"dce52edc-a5c8-4d1f-93e1-2fac29d5eec0"}
152052024-02-06T22:22:08.305Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:07.455875027Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698}
152062024-02-06T22:22:08.305ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152072024-02-06T22:22:08.305ZLoading write count information from file "/tmp/test_repair_verify.data"
152082024-02-06T22:22:08.305ZRun Repair workload
152092024-02-06T22:22:08.305Z01/30 Flush
152102024-02-06T22:22:08.305Z02/30 Write block 28 len 8192 data: 10 8
152112024-02-06T22:22:08.305Z03/30 Read block 474 len 16384
152122024-02-06T22:22:08.305Z04/30 Write block 28 len 12288 data: 11 9 8
152132024-02-06T22:22:08.305Z05/30 Read block 81 len 40960
152142024-02-06T22:22:08.306Z06/30 Read block 16 len 24576
152152024-02-06T22:22:08.306Z07/30 Read block 487 len 12288
152162024-02-06T22:22:08.306Z08/30 Read block 535 len 16384
152172024-02-06T22:22:08.306Z09/30 Write block 141 len 36864 data: 5 6 8 9 10 13 14 16 18
152182024-02-06T22:22:08.306Z10/30 Flush
152192024-02-06T22:22:08.306Z11/30 Flush
152202024-02-06T22:22:08.306Z12/30 Write block 495 len 8192 data: 11 13
152212024-02-06T22:22:08.331Z13/30 Write block 318 len 28672 data: 14 12 10 11 14 11 11
152222024-02-06T22:22:08.332Z14/30 Write block 313 len 16384 data: 15 15 17 16
152232024-02-06T22:22:08.332Z15/30 Read block 276 len 24576
152242024-02-06T22:22:08.332Z16/30 Write block 252 len 32768 data: 9 9 11 9 6 8 9 10
152252024-02-06T22:22:08.332Z17/30 Read block 54 len 24576
152262024-02-06T22:22:08.332Z18/30 Flush
152272024-02-06T22:22:08.332Z19/30 Write block 155 len 40960 data: 18 15 17 13 14 12 10 14 12 12
152282024-02-06T22:22:08.332Z20/30 Flush
152292024-02-06T22:22:08.332Z21/30 Flush
152302024-02-06T22:22:08.332Z22/30 Write block 579 len 24576 data: 13 12 8 6 7 10
152312024-02-06T22:22:08.333Z23/30 Read block 263 len 32768
152322024-02-06T22:22:08.333Z24/30 Read block 114 len 8192
152332024-02-06T22:22:08.333Z25/30 Read block 540 len 36864
152342024-02-06T22:22:08.333Z26/30 Write block 517 len 8192 data: 13 10
152352024-02-06T22:22:08.333Z27/30 Write block 526 len 32768 data: 10 10 10 13 15 16 14 17
152362024-02-06T22:22:08.333Z28/30 Write block 60 len 28672 data: 13 10 8 8 6 7 8
152372024-02-06T22:22:08.334Z29/30 Write block 283 len 28672 data: 7 8 6 6 5 11 12
152382024-02-06T22:22:08.334Z30/30 Write block 446 len 4096 data: 8
152392024-02-06T22:22:08.334Z----------------------------------------------------------------
152402024-02-06T22:22:08.334Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
152412024-02-06T22:22:08.334ZStates: Active Active Active
152422024-02-06T22:22:08.334ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
152432024-02-06T22:22:08.334Z 1 Acked 1000 Flush 0 Done Sent Done false
152442024-02-06T22:22:08.335Z 2 Acked 1001 Write 2 Done Sent Done false
152452024-02-06T22:22:08.335Z 3 Acked 1002 Read 4 Done Sent Done false
152462024-02-06T22:22:08.335Z 4 Acked 1003 Write 3 Done Sent Done false
152472024-02-06T22:22:08.335Z 5 Acked 1004 Read 10 Done Sent Done false
152482024-02-06T22:22:08.335Z 6 Acked 1005 Read 6 Done Sent Done false
152492024-02-06T22:22:08.335Z 7 Acked 1006 Read 3 Done Sent Done false
152502024-02-06T22:22:08.335Z 8 Acked 1007 Read 4 Done Sent Done false
152512024-02-06T22:22:08.336Z 9 Acked 1008 Write 9 Done Sent Done false
152522024-02-06T22:22:08.336Z 10 Acked 1009 Flush 0 Done Sent Done false
152532024-02-06T22:22:08.336Z 11 Acked 1010 Flush 0 Done Sent Done false
152542024-02-06T22:22:08.336Z 12 Acked 1011 Write 2 Done Sent Done false
152552024-02-06T22:22:08.336Z 13 Acked 1012 Write 7 Done Sent Done false
152562024-02-06T22:22:08.336Z 14 Acked 1013 Write 4 Done Sent Done false
152572024-02-06T22:22:08.336Z 15 Acked 1014 Read 6 Done Sent Done false
152582024-02-06T22:22:08.337Z 16 Acked 1015 Write 8 Done Sent Done false
152592024-02-06T22:22:08.337Z 17 Acked 1016 Read 6 Done Sent Done false
152602024-02-06T22:22:08.337Z 18 Acked 1017 Flush 0 Done Sent Done false
152612024-02-06T22:22:08.337Z 19 Acked 1018 Write 10 Done Sent Done false
152622024-02-06T22:22:08.337Z 20 Acked 1019 Flush 0 Done Sent Done false
152632024-02-06T22:22:08.337Z 21 Acked 1020 Flush 0 Done Sent Done false
152642024-02-06T22:22:08.337Z 22 Acked 1021 Write 6 Done Sent Done false
152652024-02-06T22:22:08.338Z 23 Acked 1022 Read 8 Done Sent Done false
152662024-02-06T22:22:08.338Z 24 Acked 1023 Read 2 Done Sent Done false
152672024-02-06T22:22:08.338Z 25 Acked 1024 Read 9 Done Sent Done false
152682024-02-06T22:22:08.338Z 26 Acked 1025 Write 2 Done Sent Done false
152692024-02-06T22:22:08.338Z 27 Acked 1026 Write 8 Done Sent Done false
152702024-02-06T22:22:08.338Z 28 Acked 1027 Write 7 Done Sent Done false
152712024-02-06T22:22:08.338Z 29 Acked 1028 Write 7 Sent Sent Sent false
152722024-02-06T22:22:08.339Z 30 Acked 1029 Write 1 Sent Sent Sent false
152732024-02-06T22:22:08.339Z STATES DS:0 DS:1 DS:2 TOTAL
152742024-02-06T22:22:08.339Z New 0 0 0 0
152752024-02-06T22:22:08.339Z Sent 2 30 2 34
152762024-02-06T22:22:08.339Z Done 28 0 28 56
152772024-02-06T22:22:08.339Z Skipped 0 0 0 0
152782024-02-06T22:22:08.339Z Error 0 0 0 0
152792024-02-06T22:22:08.340ZLast Flush: 1020 0 1020
152802024-02-06T22:22:08.340ZDownstairs last five completed:
152812024-02-06T22:22:08.340ZUpstairs last five completed: 30 29 28 27 26
152822024-02-06T22:22:08.340Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:07.485796124Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1698}
152832024-02-06T22:22:08.340ZWrote out file "/tmp/test_repair_verify.data"
152842024-02-06T22:22:08.340Z
152852024-02-06T22:22:08.341ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
152862024-02-06T22:22:08.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152872024-02-06T22:22:08.367Z 0 000-019 130 130 130 246 246 246 F F F
152882024-02-06T22:22:08.367Z 1 020-039 140 138 140 262 258 262 F F F <---
152892024-02-06T22:22:08.367Z 2 040-059 138 138 138 259 259 259 F F F
152902024-02-06T22:22:08.368Z 3 060-079 139 139 139 260 260 260 T F T <---
152912024-02-06T22:22:08.368Z 4 080-099 139 139 139 260 260 260 F F F
152922024-02-06T22:22:08.368Z 5 100-119 133 133 133 249 249 249 F F F
152932024-02-06T22:22:08.368Z 6 120-139 138 138 138 259 259 259 F F F
152942024-02-06T22:22:08.368Z 7 140-159 140 136 140 265 255 265 F F F <---
152952024-02-06T22:22:08.368Z 8 160-179 140 138 140 265 258 265 F F F <---
152962024-02-06T22:22:08.368Z 9 180-199 137 137 137 256 256 256 F F F
152972024-02-06T22:22:08.368Z 10 200-219 138 138 138 259 259 259 F F F
152982024-02-06T22:22:08.368Z 11 220-239 138 138 138 259 259 259 F F F
152992024-02-06T22:22:08.369Z 12 240-259 140 139 140 264 260 264 F F F <---
153002024-02-06T22:22:08.369Z 13 260-279 138 138 138 258 258 258 F F F
153012024-02-06T22:22:08.369Z 14 280-299 137 137 137 256 256 256 T F T <---
153022024-02-06T22:22:08.369Z 15 300-319 140 132 140 264 248 264 F F F <---
153032024-02-06T22:22:08.369Z 16 320-339 140 132 140 264 248 264 F F F <---
153042024-02-06T22:22:08.369Z 17 340-359 137 137 137 256 256 256 F F F
153052024-02-06T22:22:08.370Z 18 360-379 138 138 138 258 258 258 F F F
153062024-02-06T22:22:08.370Z 19 380-399 118 118 118 222 222 222 F F F
153072024-02-06T22:22:08.370Z 20 400-419 130 130 130 246 246 246 F F F
153082024-02-06T22:22:08.370Z 21 420-439 139 139 139 260 260 260 F F F
153092024-02-06T22:22:08.370Z 22 440-459 139 139 139 260 260 260 T F T <---
153102024-02-06T22:22:08.370Z 23 460-479 136 136 136 255 255 255 F F F
153112024-02-06T22:22:08.370Z 24 480-499 140 126 140 264 240 264 F F F <---
153122024-02-06T22:22:08.370Z 25 500-519 138 138 138 257 257 257 T F T <---
153132024-02-06T22:22:08.371Z 26 520-539 138 138 138 259 259 259 T F T <---
153142024-02-06T22:22:08.371Z 27 540-559 136 136 136 254 254 254 F F F
153152024-02-06T22:22:08.371Z 28 560-579 135 135 135 253 253 253 T F T <---
153162024-02-06T22:22:08.371Z 29 580-599 138 138 138 259 259 259 T F T <---
153172024-02-06T22:22:08.371ZMax gen: 140, Max flush: 265
153182024-02-06T22:22:08.371ZError: Difference in extent metadata found!
153192024-02-06T22:22:08.371ZA Difference in extent metadata is expected here
153202024-02-06T22:22:08.372ZOn loop 70
153212024-02-06T22:22:08.372Z
153222024-02-06T22:22:08.372ZVerifying data now
153232024-02-06T22:22:13.011ZNo change after verify
153242024-02-06T22:22:13.011ZLoop: 70 Downstairs dump after verify (and repair):
153252024-02-06T22:22:13.025ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153262024-02-06T22:22:13.025Z 0 000-019 130 130 130 246 246 246 F F F
153272024-02-06T22:22:13.025Z 1 020-039 140 140 140 262 262 262 F F F
153282024-02-06T22:22:13.025Z 2 040-059 138 138 138 259 259 259 F F F
153292024-02-06T22:22:13.025Z 3 060-079 141 141 141 266 266 266 F F F
153302024-02-06T22:22:13.026Z 4 080-099 139 139 139 260 260 260 F F F
153312024-02-06T22:22:13.026Z 5 100-119 133 133 133 249 249 249 F F F
153322024-02-06T22:22:13.026Z 6 120-139 138 138 138 259 259 259 F F F
153332024-02-06T22:22:13.026Z 7 140-159 140 140 140 265 265 265 F F F
153342024-02-06T22:22:13.026Z 8 160-179 140 140 140 265 265 265 F F F
153352024-02-06T22:22:13.026Z 9 180-199 137 137 137 256 256 256 F F F
153362024-02-06T22:22:13.027Z 10 200-219 138 138 138 259 259 259 F F F
153372024-02-06T22:22:13.027Z 11 220-239 138 138 138 259 259 259 F F F
153382024-02-06T22:22:13.027Z 12 240-259 140 140 140 264 264 264 F F F
153392024-02-06T22:22:13.027Z 13 260-279 138 138 138 258 258 258 F F F
153402024-02-06T22:22:13.033Z 14 280-299 141 141 141 266 266 266 F F F
153412024-02-06T22:22:13.033Z 15 300-319 140 140 140 264 264 264 F F F
153422024-02-06T22:22:13.033Z 16 320-339 140 140 140 264 264 264 F F F
153432024-02-06T22:22:13.033Z 17 340-359 137 137 137 256 256 256 F F F
153442024-02-06T22:22:13.034Z 18 360-379 138 138 138 258 258 258 F F F
153452024-02-06T22:22:13.034Z 19 380-399 118 118 118 222 222 222 F F F
153462024-02-06T22:22:13.034Z 20 400-419 130 130 130 246 246 246 F F F
153472024-02-06T22:22:13.034Z 21 420-439 139 139 139 260 260 260 F F F
153482024-02-06T22:22:13.034Z 22 440-459 141 141 141 266 266 266 F F F
153492024-02-06T22:22:13.034Z 23 460-479 136 136 136 255 255 255 F F F
153502024-02-06T22:22:13.035Z 24 480-499 140 140 140 264 264 264 F F F
153512024-02-06T22:22:13.035Z 25 500-519 141 141 141 266 266 266 F F F
153522024-02-06T22:22:13.035Z 26 520-539 141 141 141 266 266 266 F F F
153532024-02-06T22:22:13.035Z 27 540-559 136 136 136 254 254 254 F F F
153542024-02-06T22:22:13.035Z 28 560-579 141 141 141 266 266 266 F F F
153552024-02-06T22:22:13.035Z 29 580-599 141 141 141 266 266 266 F F F
153562024-02-06T22:22:13.035ZMax gen: 141, Max flush: 266
153572024-02-06T22:22:13.036Z
153582024-02-06T22:22:13.036ZBegin loop 71
153592024-02-06T22:22:13.036ZDownstairs to restart: 1
153602024-02-06T22:22:13.047Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231247636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706}
153612024-02-06T22:22:13.048ZCrucible runtime is spawned
153622024-02-06T22:22:13.060Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231548755Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706}
153632024-02-06T22:22:13.060Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231595549Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706}
153642024-02-06T22:22:13.060Z{"msg":"Crucible 44adfd81-0a41-4253-9323-b89f04f100c4 has session id: bf4818aa-3efb-4663-94bc-a9ab987295af","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231615286Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153652024-02-06T22:22:13.061Z{"msg":"Upstairs opts: Upstairs UUID: 44adfd81-0a41-4253-9323-b89f04f100c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231646451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153662024-02-06T22:22:13.061Z{"msg":"Crucible stats registered with UUID: 44adfd81-0a41-4253-9323-b89f04f100c4","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231664309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153672024-02-06T22:22:13.061Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231691196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706}
153682024-02-06T22:22:13.061Z{"msg":"44adfd81-0a41-4253-9323-b89f04f100c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231707153Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153692024-02-06T22:22:13.061Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.23173352Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153702024-02-06T22:22:13.061Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231768745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153712024-02-06T22:22:13.061Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231810709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153722024-02-06T22:22:13.061Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231860673Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153732024-02-06T22:22:13.062Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231906596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153742024-02-06T22:22:13.062Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231942602Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153752024-02-06T22:22:13.062Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.231981766Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153762024-02-06T22:22:13.062Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232018001Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153772024-02-06T22:22:13.062Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232066185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153782024-02-06T22:22:13.062Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.23210135Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153792024-02-06T22:22:13.062Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232150853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153802024-02-06T22:22:13.062Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:12.232350096Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153812024-02-06T22:22:13.062Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:12.232391981Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153822024-02-06T22:22:13.063Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:12.232410868Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153832024-02-06T22:22:13.063Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232424776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153842024-02-06T22:22:13.063Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232438595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153852024-02-06T22:22:13.063Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232595303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153862024-02-06T22:22:13.063Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232873846Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153872024-02-06T22:22:13.063Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232927438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153882024-02-06T22:22:13.063Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232957734Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153892024-02-06T22:22:13.064Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.232983231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153902024-02-06T22:22:13.064Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.233502721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153912024-02-06T22:22:13.064Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.233643442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153922024-02-06T22:22:13.064Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.234056086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153932024-02-06T22:22:13.064Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.234088901Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153942024-02-06T22:22:13.064Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.234104889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153952024-02-06T22:22:13.064Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.234138645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153962024-02-06T22:22:13.064Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.23417162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153972024-02-06T22:22:13.064Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:12.234205736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153982024-02-06T22:22:14.047Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.232853546Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
153992024-02-06T22:22:14.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.233066758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154002024-02-06T22:22:14.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.234028968Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154012024-02-06T22:22:14.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.234073522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154022024-02-06T22:22:14.170Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.234541548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154032024-02-06T22:22:14.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235059018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154042024-02-06T22:22:14.170Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235110591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154052024-02-06T22:22:14.170Z{"msg":"[0]R flush_numbers[0..12]: [246, 262, 259, 266, 260, 249, 259, 265, 265, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235133768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154062024-02-06T22:22:14.170Z{"msg":"[0]R generation[0..12]: [130, 140, 138, 141, 139, 133, 138, 140, 140, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235157475Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154072024-02-06T22:22:14.170Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235172923Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154082024-02-06T22:22:14.170Z{"msg":"[1]R flush_numbers[0..12]: [246, 262, 259, 266, 260, 249, 259, 265, 265, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235187471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154092024-02-06T22:22:14.170Z{"msg":"[1]R generation[0..12]: [130, 140, 138, 141, 139, 133, 138, 140, 140, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235208588Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154102024-02-06T22:22:14.170Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235223146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154112024-02-06T22:22:14.170Z{"msg":"[2]R flush_numbers[0..12]: [246, 262, 259, 266, 260, 249, 259, 265, 265, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235237874Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154122024-02-06T22:22:14.170Z{"msg":"[2]R generation[0..12]: [130, 140, 138, 141, 139, 133, 138, 140, 140, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235251732Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154132024-02-06T22:22:14.170Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235273169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154142024-02-06T22:22:14.170Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235286958Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154152024-02-06T22:22:14.170Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
154162024-02-06T22:22:14.170Z,"time":"2024-02-06T22:22:13.235301525Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154172024-02-06T22:22:14.171Z{"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235316823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154182024-02-06T22:22:14.171Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235337491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154192024-02-06T22:22:14.171Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235351689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154202024-02-06T22:22:14.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235363867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154212024-02-06T22:22:14.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235389154Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154222024-02-06T22:22:14.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235402862Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154232024-02-06T22:22:14.171Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235422799Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154242024-02-06T22:22:14.171Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235435587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154252024-02-06T22:22:14.171Z{"msg":"44adfd81-0a41-4253-9323-b89f04f100c4 is now active with session: bf4818aa-3efb-4663-94bc-a9ab987295af","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235448866Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154262024-02-06T22:22:14.171Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235460864Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706,"session_id":"bf4818aa-3efb-4663-94bc-a9ab987295af"}
154272024-02-06T22:22:14.171Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:13.235473272Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706}
154282024-02-06T22:22:14.171ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154292024-02-06T22:22:14.171ZLoading write count information from file "/tmp/test_repair_verify.data"
154302024-02-06T22:22:14.171ZRun Repair workload
154312024-02-06T22:22:14.171Z01/30 Write block 422 len 4096 data: 11
154322024-02-06T22:22:14.171Z02/30 Write block 306 len 20480 data: 14 12 13 11 12
154332024-02-06T22:22:14.171Z03/30 Read block 236 len 16384
154342024-02-06T22:22:14.171Z04/30 Write block 380 len 4096 data: 12
154352024-02-06T22:22:14.185Z05/30 Flush
154362024-02-06T22:22:14.185Z06/30 Write block 74 len 4096 data: 12
154372024-02-06T22:22:14.185Z07/30 Flush
154382024-02-06T22:22:14.185Z08/30 Write block 332 len 12288 data: 7 10 13
154392024-02-06T22:22:14.185Z09/30 Read block 178 len 36864
154402024-02-06T22:22:14.185Z10/30 Write block 116 len 36864 data: 11 13 13 11 10 11 12 14 14
154412024-02-06T22:22:14.185Z11/30 Read block 222 len 40960
154422024-02-06T22:22:14.185Z12/30 Write block 341 len 8192 data: 12 16
154432024-02-06T22:22:14.185Z13/30 Write block 278 len 32768 data: 13 13 13 12 10 8 9 7
154442024-02-06T22:22:14.185Z14/30 Read block 225 len 4096
154452024-02-06T22:22:14.185Z15/30 Read block 292 len 16384
154462024-02-06T22:22:14.185Z16/30 Write block 258 len 36864 data: 10 11 12 17 18 19 17 18 19
154472024-02-06T22:22:14.185Z17/30 Flush
154482024-02-06T22:22:14.185Z18/30 Write block 111 len 28672 data: 12 11 12 14 14 12 14
154492024-02-06T22:22:14.185Z19/30 Read block 218 len 40960
154502024-02-06T22:22:14.185Z20/30 Write block 127 len 8192 data: 15 16
154512024-02-06T22:22:14.185Z21/30 Write block 339 len 28672 data: 12 13 13 17 12 14 11
154522024-02-06T22:22:14.185Z22/30 Read block 335 len 32768
154532024-02-06T22:22:14.185Z23/30 Read block 207 len 36864
154542024-02-06T22:22:14.185Z24/30 Read block 479 len 32768
154552024-02-06T22:22:14.185Z25/30 Write block 54 len 32768 data: 23 21 20 16 14 13 14 11
154562024-02-06T22:22:14.185Z26/30 Read block 463 len 20480
154572024-02-06T22:22:14.186Z27/30 Write block 351 len 4096 data: 20
154582024-02-06T22:22:14.186Z28/30 Write block 167 len 16384 data: 12 15 16 16
154592024-02-06T22:22:14.186Z29/30 Write block 370 len 28672 data: 11 9 7 10 11 10 11
154602024-02-06T22:22:14.186Z30/30 Read block 22 len 32768
154612024-02-06T22:22:14.186Z----------------------------------------------------------------
154622024-02-06T22:22:14.186Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
154632024-02-06T22:22:14.186ZStates: Active Active Active
154642024-02-06T22:22:14.186ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
154652024-02-06T22:22:14.186Z 1 Acked 1000 Write 1 Done Done Done false
154662024-02-06T22:22:14.186Z 2 Acked 1001 Write 5 Done Sent Done false
154672024-02-06T22:22:14.186Z 3 Acked 1002 Read 4 Done Sent Done false
154682024-02-06T22:22:14.186Z 4 Acked 1003 Write 1 Done Sent Done false
154692024-02-06T22:22:14.186Z 5 Acked 1004 Flush 0 Done Sent Done false
154702024-02-06T22:22:14.186Z 6 Acked 1005 Write 1 Done Sent Done false
154712024-02-06T22:22:14.186Z 7 Acked 1006 Flush 0 Done Sent Done false
154722024-02-06T22:22:14.186Z 8 Acked 1007 Write 3 Done Sent Done false
154732024-02-06T22:22:14.186Z 9 Acked 1008 Read 9 Done Sent Done false
154742024-02-06T22:22:14.186Z 10 Acked 1009 Write 9 Done Sent Done false
154752024-02-06T22:22:14.186Z 11 Acked 1010 Read 10 Done Sent Done false
154762024-02-06T22:22:14.186Z 12 Acked 1011 Write 2 Done Sent Done false
154772024-02-06T22:22:14.186Z 13 Acked 1012 Write 8 Done Sent Done false
154782024-02-06T22:22:14.186Z 14 Acked 1013 Read 1 Done Sent Done false
154792024-02-06T22:22:14.186Z 15 Acked 1014 Read 4 Done Sent Done false
154802024-02-06T22:22:14.186Z 16 Acked 1015 Write 9 Done Sent Done false
154812024-02-06T22:22:14.186Z 17 Acked 1016 Flush 0 Done Sent Done false
154822024-02-06T22:22:14.187Z 18 Acked 1017 Write 7 Done Sent Done false
154832024-02-06T22:22:14.187Z 19 Acked 1018 Read 10 Done Sent Done false
154842024-02-06T22:22:14.187Z 20 Acked 1019 Write 2 Done Sent Done false
154852024-02-06T22:22:14.187Z 21 Acked 1020 Write 7 Done Sent Done false
154862024-02-06T22:22:14.187Z 22 Acked 1021 Read 8 Done Sent Done false
154872024-02-06T22:22:14.187Z 23 Acked 1022 Read 9 Done Sent Done false
154882024-02-06T22:22:14.187Z 24 Acked 1023 Read 8 Done Sent Done false
154892024-02-06T22:22:14.187Z 25 Acked 1024 Write 8 Done Sent Done false
154902024-02-06T22:22:14.187Z 26 Acked 1025 Read 5 Done Sent Done false
154912024-02-06T22:22:14.187Z 27 Acked 1026 Write 1 Done Sent Done false
154922024-02-06T22:22:14.187Z 28 Acked 1027 Write 4 Done Sent Done false
154932024-02-06T22:22:14.187Z 29 Acked 1028 Write 7 Done Sent Done false
154942024-02-06T22:22:14.187Z 30 Acked 1029 Read 8 Done Sent Done false
154952024-02-06T22:22:14.187Z STATES DS:0 DS:1 DS:2 TOTAL
154962024-02-06T22:22:14.187Z New 0 0 0 0
154972024-02-06T22:22:14.187Z Sent 0 29 0 29
154982024-02-06T22:22:14.187Z Done 30 1 30 61
154992024-02-06T22:22:14.187Z Skipped 0 0 0 0
155002024-02-06T22:22:14.187Z Error 0 0 0 0
155012024-02-06T22:22:14.187ZLast Flush: 1016 0 1016
155022024-02-06T22:22:14.187ZDownstairs last five completed:
155032024-02-06T22:22:14.187ZUpstairs last five completed: 30 29 28 27 26
155042024-02-06T22:22:14.187Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:13.268271461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1706}
155052024-02-06T22:22:14.188ZWrote out file "/tmp/test_repair_verify.data"
155062024-02-06T22:22:14.188Z
155072024-02-06T22:22:14.188ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155082024-02-06T22:22:14.188ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155092024-02-06T22:22:14.188Z 0 000-019 130 130 130 246 246 246 F F F
155102024-02-06T22:22:14.188Z 1 020-039 140 140 140 262 262 262 F F F
155112024-02-06T22:22:14.188Z 2 040-059 138 138 138 259 259 259 T F T <---
155122024-02-06T22:22:14.188Z 3 060-079 142 141 142 268 266 268 T F T <---
155132024-02-06T22:22:14.188Z 4 080-099 139 139 139 260 260 260 F F F
155142024-02-06T22:22:14.188Z 5 100-119 142 133 142 269 249 269 T F T <---
155152024-02-06T22:22:14.188Z 6 120-139 142 138 142 269 259 269 T F T <---
155162024-02-06T22:22:14.188Z 7 140-159 140 140 140 265 265 265 F F F
155172024-02-06T22:22:14.188Z 8 160-179 140 140 140 265 265 265 T F T <---
155182024-02-06T22:22:14.188Z 9 180-199 137 137 137 256 256 256 F F F
155192024-02-06T22:22:14.188Z 10 200-219 138 138 138 259 259 259 F F F
155202024-02-06T22:22:14.188Z 11 220-239 138 138 138 259 259 259 F F F
155212024-02-06T22:22:14.188Z 12 240-259 142 140 142 269 264 269 F F F <---
155222024-02-06T22:22:14.188Z 13 260-279 142 138 142 269 258 269 F F F <---
155232024-02-06T22:22:14.188Z 14 280-299 142 141 142 269 266 269 F F F <---
155242024-02-06T22:22:14.188Z 15 300-319 142 140 142 267 264 267 F F F <---
155252024-02-06T22:22:14.188Z 16 320-339 142 140 142 269 264 269 T F T <---
155262024-02-06T22:22:14.188Z 17 340-359 142 137 142 269 256 269 T F T <---
155272024-02-06T22:22:14.188Z 18 360-379 138 138 138 258 258 258 T F T <---
155282024-02-06T22:22:14.188Z 19 380-399 142 118 142 267 222 267 F F F <---
155292024-02-06T22:22:14.188Z 20 400-419 130 130 130 246 246 246 F F F
155302024-02-06T22:22:14.189Z 21 420-439 142 139 142 267 260 267 F T F <---
155312024-02-06T22:22:14.189Z 22 440-459 141 141 141 266 266 266 F F F
155322024-02-06T22:22:14.189Z 23 460-479 136 136 136 255 255 255 F F F
155332024-02-06T22:22:14.189Z 24 480-499 140 140 140 264 264 264 F F F
155342024-02-06T22:22:14.189Z 25 500-519 141 141 141 266 266 266 F F F
155352024-02-06T22:22:14.198Z 26 520-539 141 141 141 266 266 266 F F F
155362024-02-06T22:22:14.198Z 27 540-559 136 136 136 254 254 254 F F F
155372024-02-06T22:22:14.198Z 28 560-579 141 141 141 266 266 266 F F F
155382024-02-06T22:22:14.198Z 29 580-599 141 141 141 266 266 266 F F F
155392024-02-06T22:22:14.198ZMax gen: 142, Max flush: 269
155402024-02-06T22:22:14.198ZError: Difference in extent metadata found!
155412024-02-06T22:22:14.198ZA Difference in extent metadata is expected here
155422024-02-06T22:22:14.198ZOn loop 71
155432024-02-06T22:22:14.198Z
155442024-02-06T22:22:14.198ZVerifying data now
155452024-02-06T22:22:18.801ZNo change after verify
155462024-02-06T22:22:18.801ZLoop: 71 Downstairs dump after verify (and repair):
155472024-02-06T22:22:18.816ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155482024-02-06T22:22:18.816Z 0 000-019 130 130 130 246 246 246 F F F
155492024-02-06T22:22:18.816Z 1 020-039 140 140 140 262 262 262 F F F
155502024-02-06T22:22:18.816Z 2 040-059 143 143 143 270 270 270 F F F
155512024-02-06T22:22:18.816Z 3 060-079 143 143 143 270 270 270 F F F
155522024-02-06T22:22:18.816Z 4 080-099 139 139 139 260 260 260 F F F
155532024-02-06T22:22:18.816Z 5 100-119 143 143 143 270 270 270 F F F
155542024-02-06T22:22:18.816Z 6 120-139 143 143 143 270 270 270 F F F
155552024-02-06T22:22:18.817Z 7 140-159 140 140 140 265 265 265 F F F
155562024-02-06T22:22:18.817Z 8 160-179 143 143 143 270 270 270 F F F
155572024-02-06T22:22:18.817Z 9 180-199 137 137 137 256 256 256 F F F
155582024-02-06T22:22:18.823Z 10 200-219 138 138 138 259 259 259 F F F
155592024-02-06T22:22:18.823Z 11 220-239 138 138 138 259 259 259 F F F
155602024-02-06T22:22:18.823Z 12 240-259 142 142 142 269 269 269 F F F
155612024-02-06T22:22:18.823Z 13 260-279 142 142 142 269 269 269 F F F
155622024-02-06T22:22:18.823Z 14 280-299 142 142 142 269 269 269 F F F
155632024-02-06T22:22:18.824Z 15 300-319 142 142 142 267 267 267 F F F
155642024-02-06T22:22:18.824Z 16 320-339 143 143 143 270 270 270 F F F
155652024-02-06T22:22:18.824Z 17 340-359 143 143 143 270 270 270 F F F
155662024-02-06T22:22:18.824Z 18 360-379 143 143 143 270 270 270 F F F
155672024-02-06T22:22:18.824Z 19 380-399 142 142 142 267 267 267 F F F
155682024-02-06T22:22:18.824Z 20 400-419 130 130 130 246 246 246 F F F
155692024-02-06T22:22:18.824Z 21 420-439 142 142 142 267 267 267 F F F
155702024-02-06T22:22:18.824Z 22 440-459 141 141 141 266 266 266 F F F
155712024-02-06T22:22:18.824Z 23 460-479 136 136 136 255 255 255 F F F
155722024-02-06T22:22:18.824Z 24 480-499 140 140 140 264 264 264 F F F
155732024-02-06T22:22:18.825Z 25 500-519 141 141 141 266 266 266 F F F
155742024-02-06T22:22:18.825Z 26 520-539 141 141 141 266 266 266 F F F
155752024-02-06T22:22:18.825Z 27 540-559 136 136 136 254 254 254 F F F
155762024-02-06T22:22:18.825Z 28 560-579 141 141 141 266 266 266 F F F
155772024-02-06T22:22:18.825Z 29 580-599 141 141 141 266 266 266 F F F
155782024-02-06T22:22:18.825ZMax gen: 143, Max flush: 270
155792024-02-06T22:22:18.825Z
155802024-02-06T22:22:18.825ZBegin loop 72
155812024-02-06T22:22:18.825ZDownstairs to restart: 1
155822024-02-06T22:22:18.840Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.022861417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714}
155832024-02-06T22:22:18.840Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023146699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714}
155842024-02-06T22:22:18.840Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023165176Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714}
155852024-02-06T22:22:18.856Z{"msg":"Crucible 8280b1f6-ec63-448e-a762-3f90b1817944 has session id: ff4e5ae7-b7b1-4be5-9de6-f807832273cc","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023179204Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155862024-02-06T22:22:18.857Z{"msg":"Upstairs opts: Upstairs UUID: 8280b1f6-ec63-448e-a762-3f90b1817944, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023219219Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155872024-02-06T22:22:18.857Z{"msg":"Crucible stats registered with UUID: 8280b1f6-ec63-448e-a762-3f90b1817944","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
155882024-02-06T22:22:18.857Z2024-02-06T22:22:18.023245155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155892024-02-06T22:22:18.857Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.02335584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714}
155902024-02-06T22:22:18.857Z{"msg":"8280b1f6-ec63-448e-a762-3f90b1817944 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.02350715Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155912024-02-06T22:22:18.857Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023533996Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155922024-02-06T22:22:18.857Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023550224Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155932024-02-06T22:22:18.858Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023569112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155942024-02-06T22:22:18.858Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023625954Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155952024-02-06T22:22:18.858Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023674988Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155962024-02-06T22:22:18.858Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023701754Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155972024-02-06T22:22:18.858Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023725371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155982024-02-06T22:22:18.858Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023759976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
155992024-02-06T22:22:18.858Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023798321Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156002024-02-06T22:22:18.859Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023873241Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156012024-02-06T22:22:18.859Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.023919564Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156022024-02-06T22:22:18.859Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:18.02395152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156032024-02-06T22:22:18.859Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:18.023989425Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156042024-02-06T22:22:18.859Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:18.024128136Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156052024-02-06T22:22:18.859Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.02417603Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156062024-02-06T22:22:18.859Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.024205356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156072024-02-06T22:22:18.859Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.024234622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156082024-02-06T22:22:18.860Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.024577155Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156092024-02-06T22:22:18.860Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.024602032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156102024-02-06T22:22:18.860Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.02461685Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156112024-02-06T22:22:18.860Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.024640597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156122024-02-06T22:22:18.860Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.025198362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156132024-02-06T22:22:18.860Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.025247795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156142024-02-06T22:22:18.860Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.025812939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156152024-02-06T22:22:18.860Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.025850483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156162024-02-06T22:22:18.860Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.025867191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156172024-02-06T22:22:18.861Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.02588041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156182024-02-06T22:22:18.861Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.025902287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156192024-02-06T22:22:18.861Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:18.025916165Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156202024-02-06T22:22:19.847Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.025464923Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156212024-02-06T22:22:19.847Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.025569739Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156222024-02-06T22:22:19.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.026646283Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156232024-02-06T22:22:19.871Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.026685688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156242024-02-06T22:22:19.871Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027180231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156252024-02-06T22:22:19.871Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027731857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156262024-02-06T22:22:19.872Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027765752Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156272024-02-06T22:22:19.872Z{"msg":"[0]R flush_numbers[0..12]: [246, 262, 270, 270, 260, 270, 270, 265, 270, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027795098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156282024-02-06T22:22:19.872Z{"msg":"[0]R generation[0..12]: [130, 140, 143, 143, 139, 143, 143, 140, 143, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027833233Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156292024-02-06T22:22:19.872Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.02785849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156302024-02-06T22:22:19.872Z{"msg":"[1]R flush_numbers[0..12]: [246, 262, 270, 270, 260, 270, 270, 265, 270, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027885416Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156312024-02-06T22:22:19.872Z{"msg":"[1]R generation[0..12]: [130, 140, 143, 143, 139, 143, 143, 140, 143, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027911432Z","Wait for a query_work_queue command to finish before sending IO
156322024-02-06T22:22:19.872Zhostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156332024-02-06T22:22:19.872Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027951687Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156342024-02-06T22:22:19.872Z{"msg":"[2]R flush_numbers[0..12]: [246, 262, 270, 270, 260, 270, 270, 265, 270, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.027986602Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156352024-02-06T22:22:19.872Z{"msg":"[2]R generation[0..12]: [130, 140, 143, 143, 139, 143, 143, 140, 143, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.02800325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156362024-02-06T22:22:19.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028031856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156372024-02-06T22:22:19.873Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028045964Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156382024-02-06T22:22:19.873Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028062632Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156392024-02-06T22:22:19.873Z{"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028095648Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156402024-02-06T22:22:19.873Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028112395Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156412024-02-06T22:22:19.873Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028144521Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156422024-02-06T22:22:19.873Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028159329Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156432024-02-06T22:22:19.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028180256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156442024-02-06T22:22:19.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028208902Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156452024-02-06T22:22:19.874Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.02822384Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156462024-02-06T22:22:19.874Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028244727Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156472024-02-06T22:22:19.874Z{"msg":"8280b1f6-ec63-448e-a762-3f90b1817944 is now active with session: ff4e5ae7-b7b1-4be5-9de6-f807832273cc","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028275023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156482024-02-06T22:22:19.874Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:19.028289092Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714,"session_id":"ff4e5ae7-b7b1-4be5-9de6-f807832273cc"}
156492024-02-06T22:22:19.875Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
156502024-02-06T22:22:19.875Z,"time":"2024-02-06T22:22:19.028306999Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714}
156512024-02-06T22:22:19.875ZLoading write count information from file "/tmp/test_repair_verify.data"
156522024-02-06T22:22:19.875ZRun Repair workload
156532024-02-06T22:22:19.875Z01/30 Write block 547 len 40960 data: 9 8 7 6 9 10 11 10 9 11
156542024-02-06T22:22:19.875Z02/30 Write block 335 len 36864 data: 14 15 15 15 13 14 14 18 13
156552024-02-06T22:22:19.876Z03/30 Write block 212 len 24576 data: 11 14 16 13 10 9
156562024-02-06T22:22:19.876Z04/30 Read block 62 len 24576
156572024-02-06T22:22:19.876Z05/30 Write block 476 len 4096 data: 10
156582024-02-06T22:22:19.876Z06/30 Read block 0 len 24576
156592024-02-06T22:22:19.876Z07/30 Write block 537 len 4096 data: 10
156602024-02-06T22:22:19.876Z08/30 Read block 166 len 4096
156612024-02-06T22:22:19.876Z09/30 Read block 170 len 12288
156622024-02-06T22:22:19.877Z10/30 Read block 567 len 8192
156632024-02-06T22:22:19.877Z11/30 Flush
156642024-02-06T22:22:19.877Z12/30 Write block 499 len 36864 data: 12 8 8 9 13 11 13 14 14
156652024-02-06T22:22:19.887Z13/30 Write block 193 len 16384 data: 9 12 10 13
156662024-02-06T22:22:19.888Z14/30 Read block 150 len 36864
156672024-02-06T22:22:19.888Z15/30 Write block 516 len 8192 data: 12 14
156682024-02-06T22:22:19.888Z16/30 Read block 533 len 12288
156692024-02-06T22:22:19.888Z17/30 Read block 262 len 12288
156702024-02-06T22:22:19.889Z18/30 Read block 350 len 8192
156712024-02-06T22:22:19.889Z19/30 Write block 302 len 24576 data: 14 14 12 15 15 13
156722024-02-06T22:22:19.889Z20/30 Write block 360 len 8192 data: 14 13
156732024-02-06T22:22:19.889Z21/30 Read block 339 len 36864
156742024-02-06T22:22:19.889Z22/30 Write block 570 len 8192 data: 12 16
156752024-02-06T22:22:19.890Z23/30 Read block 337 len 28672
156762024-02-06T22:22:19.890Z24/30 Write block 161 len 4096 data: 11
156772024-02-06T22:22:19.890Z25/30 Read block 344 len 40960
156782024-02-06T22:22:19.890Z26/30 Write block 234 len 16384 data: 13 13 12 11
156792024-02-06T22:22:19.890Z27/30 Write block 413 len 32768 data: 11 10 13 14 16 16 15 15
156802024-02-06T22:22:19.891Z28/30 Read block 565 len 16384
156812024-02-06T22:22:19.891Z29/30 Read block 63 len 24576
156822024-02-06T22:22:19.891Z30/30 Write block 318 len 24576 data: 15 13 11 12 15 12
156832024-02-06T22:22:19.891Z----------------------------------------------------------------
156842024-02-06T22:22:19.892Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
156852024-02-06T22:22:19.892ZStates: Active Active Active
156862024-02-06T22:22:19.892ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
156872024-02-06T22:22:19.893Z 1 Acked 1000 Write 10 Done Done Done false
156882024-02-06T22:22:19.893Z 2 Acked 1001 Write 9 Done Done Done false
156892024-02-06T22:22:19.893Z 3 Acked 1002 Write 6 Done Sent Done false
156902024-02-06T22:22:19.893Z 4 Acked 1003 Read 6 Done Sent Done false
156912024-02-06T22:22:19.893Z 5 Acked 1004 Write 1 Done Sent Done false
156922024-02-06T22:22:19.893Z 6 Acked 1005 Read 6 Done Sent Done false
156932024-02-06T22:22:19.893Z 7 Acked 1006 Write 1 Done Sent Done false
156942024-02-06T22:22:19.893Z 8 Acked 1007 Read 1 Done Sent Done false
156952024-02-06T22:22:19.893Z 9 Acked 1008 Read 3 Done Sent Done false
156962024-02-06T22:22:19.893Z 10 Acked 1009 Read 2 Done Sent Done false
156972024-02-06T22:22:19.893Z 11 Acked 1010 Flush 0 Done Sent Done false
156982024-02-06T22:22:19.893Z 12 Acked 1011 Write 9 Done Sent Done false
156992024-02-06T22:22:19.894Z 13 Acked 1012 Write 4 Done Sent Done false
157002024-02-06T22:22:19.894Z 14 Acked 1013 Read 9 Done Sent Done false
157012024-02-06T22:22:19.894Z 15 Acked 1014 Write 2 Done Sent Done false
157022024-02-06T22:22:19.894Z 16 Acked 1015 Read 3 Done Sent Done false
157032024-02-06T22:22:19.894Z 17 Acked 1016 Read 3 Done Sent Done false
157042024-02-06T22:22:19.894Z 18 Acked 1017 Read 2 Done Sent Done false
157052024-02-06T22:22:19.894Z 19 Acked 1018 Write 6 Done Sent Done false
157062024-02-06T22:22:19.894Z 20 Acked 1019 Write 2 Done Sent Done false
157072024-02-06T22:22:19.894Z 21 Acked 1020 Read 9 Done Sent Done false
157082024-02-06T22:22:19.894Z 22 Acked 1021 Write 2 Done Sent Done false
157092024-02-06T22:22:19.894Z 23 Acked 1022 Read 7 Done Sent Done false
157102024-02-06T22:22:19.894Z 24 Acked 1023 Write 1 Done Sent Done false
157112024-02-06T22:22:19.894Z 25 Acked 1024 Read 10 Done Sent Done false
157122024-02-06T22:22:19.895Z 26 Acked 1025 Write 4 Done Sent Done false
157132024-02-06T22:22:19.895Z 27 Acked 1026 Write 8 Done Sent Done false
157142024-02-06T22:22:19.895Z 28 Acked 1027 Read 4 Done Sent Done false
157152024-02-06T22:22:19.895Z 29 Acked 1028 Read 6 Done Sent Done false
157162024-02-06T22:22:19.895Z 30 Acked 1029 Write 6 Sent Sent Sent false
157172024-02-06T22:22:19.895Z STATES DS:0 DS:1 DS:2 TOTAL
157182024-02-06T22:22:19.895Z New 0 0 0 0
157192024-02-06T22:22:19.895Z Sent 1 28 1 30
157202024-02-06T22:22:19.895Z Done 29 2 29 60
157212024-02-06T22:22:19.895Z Skipped 0 0 0 0
157222024-02-06T22:22:19.895Z Error 0 0 0 0
157232024-02-06T22:22:19.895ZLast Flush: 1010 0 1010
157242024-02-06T22:22:19.895ZDownstairs last five completed:
157252024-02-06T22:22:19.896ZUpstairs last five completed: 30 29 28 27 26
157262024-02-06T22:22:19.896Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:19.055855017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1714}
157272024-02-06T22:22:19.896ZWrote out file "/tmp/test_repair_verify.data"
157282024-02-06T22:22:19.896Z
157292024-02-06T22:22:19.896ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157302024-02-06T22:22:20.720ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157312024-02-06T22:22:20.721Z 0 000-019 130 130 130 246 246 246 F F F
157322024-02-06T22:22:20.721Z 1 020-039 140 140 140 262 262 262 F F F
157332024-02-06T22:22:20.721Z 2 040-059 143 143 143 270 270 270 F F F
157342024-02-06T22:22:20.721Z 3 060-079 143 143 143 270 270 270 F F F
157352024-02-06T22:22:20.721Z 4 080-099 139 139 139 260 260 260 F F F
157362024-02-06T22:22:20.721Z 5 100-119 143 143 143 270 270 270 F F F
157372024-02-06T22:22:20.721Z 6 120-139 143 143 143 270 270 270 F F F
157382024-02-06T22:22:20.721Z 7 140-159 140 140 140 265 265 265 F F F
157392024-02-06T22:22:20.721Z 8 160-179 143 143 143 270 270 270 T F T <---
157402024-02-06T22:22:20.721Z 9 180-199 137 137 137 256 256 256 T F T <---
157412024-02-06T22:22:20.721Z 10 200-219 144 138 144 271 259 271 F F F <---
157422024-02-06T22:22:20.722Z 11 220-239 138 138 138 259 259 259 T F T <---
157432024-02-06T22:22:20.722Z 12 240-259 142 142 142 269 269 269 F F F
157442024-02-06T22:22:20.722Z 13 260-279 142 142 142 269 269 269 F F F
157452024-02-06T22:22:20.722Z 14 280-299 142 142 142 269 269 269 F F F
157462024-02-06T22:22:20.722Z 15 300-319 142 142 142 267 267 267 T F T <---
157472024-02-06T22:22:20.722Z 16 320-339 144 143 144 271 270 271 T T T <---
157482024-02-06T22:22:20.722Z 17 340-359 144 143 144 271 270 271 F T F <---
157492024-02-06T22:22:20.722Z 18 360-379 143 143 143 270 270 270 T F T <---
157502024-02-06T22:22:20.722Z 19 380-399 142 142 142 267 267 267 F F F
157512024-02-06T22:22:20.722Z 20 400-419 130 130 130 246 246 246 T F T <---
157522024-02-06T22:22:20.722Z 21 420-439 142 142 142 267 267 267 T F T <---
157532024-02-06T22:22:20.722Z 22 440-459 141 141 141 266 266 266 F F F
157542024-02-06T22:22:20.722Z 23 460-479 144 136 144 271 255 271 F F F <---
157552024-02-06T22:22:20.723Z 24 480-499 140 140 140 264 264 264 T F T <---
157562024-02-06T22:22:20.723Z 25 500-519 141 141 141 266 266 266 T F T <---
157572024-02-06T22:22:20.723Z 26 520-539 144 141 144 271 266 271 F F F <---
157582024-02-06T22:22:20.723Z 27 540-559 144 136 144 271 254 271 F T F <---
157592024-02-06T22:22:20.723Z 28 560-579 141 141 141 266 266 266 T F T <---
157602024-02-06T22:22:20.723Z 29 580-599 141 141 141 266 266 266 F F F
157612024-02-06T22:22:20.723ZMax gen: 144, Max flush: 271
157622024-02-06T22:22:20.723ZError: Difference in extent metadata found!
157632024-02-06T22:22:20.723ZA Difference in extent metadata is expected here
157642024-02-06T22:22:20.723ZOn loop 72
157652024-02-06T22:22:20.723Z
157662024-02-06T22:22:20.728ZVerifying data now
157672024-02-06T22:22:25.666ZNo change after verify
157682024-02-06T22:22:25.666ZLoop: 72 Downstairs dump after verify (and repair):
157692024-02-06T22:22:25.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157702024-02-06T22:22:25.681Z 0 000-019 130 130 130 246 246 246 F F F
157712024-02-06T22:22:25.681Z 1 020-039 140 140 140 262 262 262 F F F
157722024-02-06T22:22:25.682Z 2 040-059 143 143 143 270 270 270 F F F
157732024-02-06T22:22:25.682Z 3 060-079 143 143 143 270 270 270 F F F
157742024-02-06T22:22:25.682Z 4 080-099 139 139 139 260 260 260 F F F
157752024-02-06T22:22:25.682Z 5 100-119 143 143 143 270 270 270 F F F
157762024-02-06T22:22:25.682Z 6 120-139 143 143 143 270 270 270 F F F
157772024-02-06T22:22:25.682Z 7 140-159 140 140 140 265 265 265 F F F
157782024-02-06T22:22:25.682Z 8 160-179 145 145 145 272 272 272 F F F
157792024-02-06T22:22:25.682Z 9 180-199 145 145 145 272 272 272 F F F
157802024-02-06T22:22:25.685Z 10 200-219 144 144 144 271 271 271 F F F
157812024-02-06T22:22:25.685Z 11 220-239 145 145 145 272 272 272 F F F
157822024-02-06T22:22:25.685Z 12 240-259 142 142 142 269 269 269 F F F
157832024-02-06T22:22:25.685Z 13 260-279 142 142 142 269 269 269 F F F
157842024-02-06T22:22:25.685Z 14 280-299 142 142 142 269 269 269 F F F
157852024-02-06T22:22:25.685Z 15 300-319 145 145 145 272 272 272 F F F
157862024-02-06T22:22:25.685Z 16 320-339 145 145 145 272 272 272 F F F
157872024-02-06T22:22:25.685Z 17 340-359 144 144 144 271 271 271 F F F
157882024-02-06T22:22:25.685Z 18 360-379 145 145 145 272 272 272 F F F
157892024-02-06T22:22:25.685Z 19 380-399 142 142 142 267 267 267 F F F
157902024-02-06T22:22:25.685Z 20 400-419 145 145 145 272 272 272 F F F
157912024-02-06T22:22:25.685Z 21 420-439 145 145 145 272 272 272 F F F
157922024-02-06T22:22:25.685Z 22 440-459 141 141 141 266 266 266 F F F
157932024-02-06T22:22:25.686Z 23 460-479 144 144 144 271 271 271 F F F
157942024-02-06T22:22:25.686Z 24 480-499 145 145 145 272 272 272 F F F
157952024-02-06T22:22:25.686Z 25 500-519 145 145 145 272 272 272 F F F
157962024-02-06T22:22:25.686Z 26 520-539 144 144 144 271 271 271 F F F
157972024-02-06T22:22:25.686Z 27 540-559 144 144 144 271 271 271 F F F
157982024-02-06T22:22:25.686Z 28 560-579 145 145 145 272 272 272 F F F
157992024-02-06T22:22:25.686Z 29 580-599 141 141 141 266 266 266 F F F
158002024-02-06T22:22:25.686ZMax gen: 145, Max flush: 272
158012024-02-06T22:22:25.686Z
158022024-02-06T22:22:25.686ZBegin loop 73
158032024-02-06T22:22:25.686ZDownstairs to restart: 2
158042024-02-06T22:22:25.706Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.88935436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722}
158052024-02-06T22:22:25.707Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.889601816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722}
158062024-02-06T22:22:25.712Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.889626133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722}
158072024-02-06T22:22:25.712Z{"msg":"Crucible a3f0d5e4-1e44-44a5-b842-f9f07866d081 has session id: 9ea17799-591d-4f6a-b17a-646ba5b8fb1e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.889659219Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158082024-02-06T22:22:25.712Z{"msg":"Crucible runtime is spawned
158092024-02-06T22:22:25.712ZUpstairs opts: Upstairs UUID: a3f0d5e4-1e44-44a5-b842-f9f07866d081, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.889700123Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158102024-02-06T22:22:25.712Z{"msg":"Crucible stats registered with UUID: a3f0d5e4-1e44-44a5-b842-f9f07866d081","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.88972636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158112024-02-06T22:22:25.712Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.889764415Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722}
158122024-02-06T22:22:25.712Z{"msg":"a3f0d5e4-1e44-44a5-b842-f9f07866d081 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.889939581Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158132024-02-06T22:22:25.712Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.889969237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158142024-02-06T22:22:25.712Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890010961Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158152024-02-06T22:22:25.712Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890048356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158162024-02-06T22:22:25.712Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890077282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158172024-02-06T22:22:25.712Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890104069Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158182024-02-06T22:22:25.712Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890120746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158192024-02-06T22:22:25.712Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890144693Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158202024-02-06T22:22:25.712Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890188787Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158212024-02-06T22:22:25.712Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890216263Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158222024-02-06T22:22:25.712Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890344576Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158232024-02-06T22:22:25.712Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890398749Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158242024-02-06T22:22:25.712Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:24.890542749Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158252024-02-06T22:22:25.712Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:24.890604241Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158262024-02-06T22:22:25.712Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:24.890623548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158272024-02-06T22:22:25.713Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890645445Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158282024-02-06T22:22:25.713Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890660593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158292024-02-06T22:22:25.713Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.890778967Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158302024-02-06T22:22:25.713Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.891223627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158312024-02-06T22:22:25.713Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.891267981Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158322024-02-06T22:22:25.713Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.891316745Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158332024-02-06T22:22:25.713Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.891359029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158342024-02-06T22:22:25.713Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.891875469Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158352024-02-06T22:22:25.713Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.891951059Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158362024-02-06T22:22:25.713Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.892397499Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158372024-02-06T22:22:25.713Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.892452381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158382024-02-06T22:22:25.713Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.892494335Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158392024-02-06T22:22:25.713Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.892511363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158402024-02-06T22:22:25.713Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.892526591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158412024-02-06T22:22:25.713Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:24.892542509Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158422024-02-06T22:22:26.708Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.891305474Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158432024-02-06T22:22:26.728Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.891507527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158442024-02-06T22:22:26.728Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.892343394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158452024-02-06T22:22:26.729Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.892381119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158462024-02-06T22:22:26.729Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.892897719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158472024-02-06T22:22:26.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893424998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158482024-02-06T22:22:26.729Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893475611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158492024-02-06T22:22:26.730Z{"msg":"[0]R flush_numbers[0..12]: [246, 262, 270, 270, 260, 270, 270, 265, 272, 272, 271, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893508837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158502024-02-06T22:22:26.730Z{"msg":"[0]R generation[0..12]: [130, 140, 143, 143, 139, 143, 143, 140, 145, 145, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893539432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158512024-02-06T22:22:26.730Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.89355833Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158522024-02-06T22:22:26.730Z{"msg":"[1]R flush_numbers[0..12]: [246, 262, 270, 270, 260, 270, 270, 265, 272, 272, 271, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893586976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158532024-02-06T22:22:26.730Z{"msg":"[1]R generation[0..12]: [130, 140, 143, 143, 139, 143, 143, 140, 145, 145, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893605804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158542024-02-06T22:22:26.730Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.89363356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"Wait for a query_work_queue command to finish before sending IO
158552024-02-06T22:22:26.731Zdownstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158562024-02-06T22:22:26.731Z{"msg":"[2]R flush_numbers[0..12]: [246, 262, 270, 270, 260, 270, 270, 265, 272, 272, 271, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893656157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158572024-02-06T22:22:26.731Z{"msg":"[2]R generation[0..12]: [130, 140, 143, 143, 139, 143, 143, 140, 145, 145, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893682723Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158582024-02-06T22:22:26.731Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.89370217Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158592024-02-06T22:22:26.731Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893727457Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158602024-02-06T22:22:26.732Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893744805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158612024-02-06T22:22:26.732Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893769631Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158622024-02-06T22:22:26.732Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893786399Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158632024-02-06T22:22:26.732Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893811756Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158642024-02-06T22:22:26.732Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893827434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158652024-02-06T22:22:26.732Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893844861Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158662024-02-06T22:22:26.733Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893861289Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158672024-02-06T22:22:26.733Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893886486Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158682024-02-06T22:22:26.733Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893902193Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158692024-02-06T22:22:26.733Z{"msg":"a3f0d5e4-1e44-44a5-b842-f9f07866d081 is now active with session: 9ea17799-591d-4f6a-b17a-646ba5b8fb1e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893919231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158702024-02-06T22:22:26.733Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893942848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722,"session_id":"9ea17799-591d-4f6a-b17a-646ba5b8fb1e"}
158712024-02-06T22:22:26.734Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:25.893958736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722}
158722024-02-06T22:22:26.734ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
158732024-02-06T22:22:26.734ZLoading write count information from file "/tmp/test_repair_verify.data"
158742024-02-06T22:22:26.734ZRun Repair workload
158752024-02-06T22:22:26.734Z01/30 Write block 279 len 16384 data: 14 14 13 11
158762024-02-06T22:22:26.735Z02/30 Read block 123 len 32768
158772024-02-06T22:22:26.735Z03/30 Write block 410 len 4096 data: 13
158782024-02-06T22:22:26.735Z04/30 Read block 437 len 20480
158792024-02-06T22:22:26.735Z05/30 Write block 348 len 8192 data: 15 15
158802024-02-06T22:22:26.753Z06/30 Write block 453 len 40960 data: 8 13 12 8 8 8 8 11 9 9
158812024-02-06T22:22:26.753Z07/30 Write block 222 len 20480 data: 15 15 15 12 12
158822024-02-06T22:22:26.753Z08/30 Read block 271 len 8192
158832024-02-06T22:22:26.754Z09/30 Read block 384 len 36864
158842024-02-06T22:22:26.754Z10/30 Write block 87 len 20480 data: 11 10 10 8 8
158852024-02-06T22:22:26.754Z11/30 Read block 4 len 36864
158862024-02-06T22:22:26.754Z12/30 Write block 112 len 4096 data: 12
158872024-02-06T22:22:26.754Z13/30 Read block 528 len 32768
158882024-02-06T22:22:26.754Z14/30 Read block 411 len 4096
158892024-02-06T22:22:26.755Z15/30 Flush
158902024-02-06T22:22:26.755Z16/30 Write block 421 len 20480 data: 13 12 10 11 11
158912024-02-06T22:22:26.755Z17/30 Read block 223 len 4096
158922024-02-06T22:22:26.755Z18/30 Write block 161 len 40960 data: 12 15 13 13 10 10 13 16 17 17
158932024-02-06T22:22:26.755Z19/30 Read block 358 len 4096
158942024-02-06T22:22:26.756Z20/30 Read block 353 len 4096
158952024-02-06T22:22:26.756Z21/30 Write block 187 len 20480 data: 17 14 13 11 11
158962024-02-06T22:22:26.756Z22/30 Read block 332 len 40960
158972024-02-06T22:22:26.756Z23/30 Read block 218 len 12288
158982024-02-06T22:22:26.756Z24/30 Read block 102 len 4096
158992024-02-06T22:22:26.756Z25/30 Read block 481 len 24576
159002024-02-06T22:22:26.757Z26/30 Write block 235 len 28672 data: 14 13 12 7 7 6 8
159012024-02-06T22:22:26.757Z27/30 Read block 156 len 36864
159022024-02-06T22:22:26.757Z28/30 Read block 527 len 36864
159032024-02-06T22:22:26.757Z29/30 Read block 104 len 28672
159042024-02-06T22:22:26.757Z30/30 Read block 506 len 24576
159052024-02-06T22:22:26.757Z----------------------------------------------------------------
159062024-02-06T22:22:26.758Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
159072024-02-06T22:22:26.758ZStates: Active Active Active
159082024-02-06T22:22:26.758ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159092024-02-06T22:22:26.758Z 1 Acked 1000 Write 4 Done Done Done false
159102024-02-06T22:22:26.758Z 2 Acked 1001 Read 8 Done Done Done false
159112024-02-06T22:22:26.759Z 3 Acked 1002 Write 1 Done Done Sent false
159122024-02-06T22:22:26.759Z 4 Acked 1003 Read 5 Done Done Sent false
159132024-02-06T22:22:26.759Z 5 Acked 1004 Write 2 Done Done Sent false
159142024-02-06T22:22:26.759Z 6 Acked 1005 Write 10 Done Done Sent false
159152024-02-06T22:22:26.759Z 7 Acked 1006 Write 5 Done Done Sent false
159162024-02-06T22:22:26.760Z 8 Acked 1007 Read 2 Done Done Sent false
159172024-02-06T22:22:26.760Z 9 Acked 1008 Read 9 Done Done Sent false
159182024-02-06T22:22:26.760Z 10 Acked 1009 Write 5 Done Done Sent false
159192024-02-06T22:22:26.760Z 11 Acked 1010 Read 9 Done Done Sent false
159202024-02-06T22:22:26.760Z 12 Acked 1011 Write 1 Done Done Sent false
159212024-02-06T22:22:26.761Z 13 Acked 1012 Read 8 Done Done Sent false
159222024-02-06T22:22:26.761Z 14 Acked 1013 Read 1 Done Done Sent false
159232024-02-06T22:22:26.761Z 15 Acked 1014 Flush 0 Done Done Sent false
159242024-02-06T22:22:26.761Z 16 Acked 1015 Write 5 Done Done Sent false
159252024-02-06T22:22:26.761Z 17 Acked 1016 Read 1 Done Done Sent false
159262024-02-06T22:22:26.761Z 18 Acked 1017 Write 10 Done Done Sent false
159272024-02-06T22:22:26.762Z 19 Acked 1018 Read 1 Done Done Sent false
159282024-02-06T22:22:26.762Z 20 Acked 1019 Read 1 Done Done Sent false
159292024-02-06T22:22:26.762Z 21 Acked 1020 Write 5 Done Done Sent false
159302024-02-06T22:22:26.762Z 22 Acked 1021 Read 10 Done Done Sent false
159312024-02-06T22:22:26.762Z 23 Acked 1022 Read 3 Done Done Sent false
159322024-02-06T22:22:26.762Z 24 Acked 1023 Read 1 Done Done Sent false
159332024-02-06T22:22:26.763Z 25 Acked 1024 Read 6 Done Done Sent false
159342024-02-06T22:22:26.763Z 26 Acked 1025 Write 7 Done Done Sent false
159352024-02-06T22:22:26.763Z 27 Acked 1026 Read 9 Done Done Sent false
159362024-02-06T22:22:26.763Z 28 Acked 1027 Read 9 Done Done Sent false
159372024-02-06T22:22:26.763Z 29 Acked 1028 Read 7 Done Done Sent false
159382024-02-06T22:22:26.763Z 30 Acked 1029 Read 6 Done Done Sent false
159392024-02-06T22:22:26.764Z STATES DS:0 DS:1 DS:2 TOTAL
159402024-02-06T22:22:26.764Z New 0 0 0 0
159412024-02-06T22:22:26.764Z Sent 0 0 28 28
159422024-02-06T22:22:26.764Z Done 30 30 2 62
159432024-02-06T22:22:26.764Z Skipped 0 0 0 0
159442024-02-06T22:22:26.764Z Error 0 0 0 0
159452024-02-06T22:22:26.764ZLast Flush: 1014 1014 0
159462024-02-06T22:22:26.765ZDownstairs last five completed:
159472024-02-06T22:22:26.765ZUpstairs last five completed: 30 29 28 27 26
159482024-02-06T22:22:26.765Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:25.923817182Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1722}
159492024-02-06T22:22:26.765ZWrote out file "/tmp/test_repair_verify.data"
159502024-02-06T22:22:26.766Z
159512024-02-06T22:22:26.785ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159522024-02-06T22:22:26.785ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159532024-02-06T22:22:26.785Z 0 000-019 130 130 130 246 246 246 F F F
159542024-02-06T22:22:26.785Z 1 020-039 140 140 140 262 262 262 F F F
159552024-02-06T22:22:26.785Z 2 040-059 143 143 143 270 270 270 F F F
159562024-02-06T22:22:26.786Z 3 060-079 143 143 143 270 270 270 F F F
159572024-02-06T22:22:26.786Z 4 080-099 146 146 139 273 273 260 F F F <---
159582024-02-06T22:22:26.786Z 5 100-119 146 146 143 273 273 270 F F F <---
159592024-02-06T22:22:26.786Z 6 120-139 143 143 143 270 270 270 F F F
159602024-02-06T22:22:26.786Z 7 140-159 140 140 140 265 265 265 F F F
159612024-02-06T22:22:26.786Z 8 160-179 145 145 145 272 272 272 T T F <---
159622024-02-06T22:22:26.786Z 9 180-199 145 145 145 272 272 272 T T F <---
159632024-02-06T22:22:26.787Z 10 200-219 144 144 144 271 271 271 F F F
159642024-02-06T22:22:26.787Z 11 220-239 146 146 145 273 273 272 T T F <---
159652024-02-06T22:22:26.787Z 12 240-259 142 142 142 269 269 269 T T F <---
159662024-02-06T22:22:26.787Z 13 260-279 146 146 142 273 273 269 F F T <---
159672024-02-06T22:22:26.787Z 14 280-299 146 146 142 273 273 269 F F T <---
159682024-02-06T22:22:26.788Z 15 300-319 145 145 145 272 272 272 F F F
159692024-02-06T22:22:26.788Z 16 320-339 145 145 145 272 272 272 F F F
159702024-02-06T22:22:26.788Z 17 340-359 146 146 144 273 273 271 F F F <---
159712024-02-06T22:22:26.788Z 18 360-379 145 145 145 272 272 272 F F F
159722024-02-06T22:22:26.788Z 19 380-399 142 142 142 267 267 267 F F F
159732024-02-06T22:22:26.788Z 20 400-419 146 146 145 273 273 272 F F F <---
159742024-02-06T22:22:26.788Z 21 420-439 145 145 145 272 272 272 T T F <---
159752024-02-06T22:22:26.788Z 22 440-459 146 146 141 273 273 266 F F F <---
159762024-02-06T22:22:26.788Z 23 460-479 146 146 144 273 273 271 F F F <---
159772024-02-06T22:22:26.789Z 24 480-499 145 145 145 272 272 272 F F F
159782024-02-06T22:22:26.789Z 25 500-519 145 145 145 272 272 272 F F F
159792024-02-06T22:22:26.789Z 26 520-539 144 144 144 271 271 271 F F F
159802024-02-06T22:22:26.789Z 27 540-559 144 144 144 271 271 271 F F F
159812024-02-06T22:22:26.789Z 28 560-579 145 145 145 272 272 272 F F F
159822024-02-06T22:22:26.789Z 29 580-599 141 141 141 266 266 266 F F F
159832024-02-06T22:22:26.789ZMax gen: 146, Max flush: 273
159842024-02-06T22:22:26.789ZError: Difference in extent metadata found!
159852024-02-06T22:22:26.789ZA Difference in extent metadata is expected here
159862024-02-06T22:22:26.789ZOn loop 73
159872024-02-06T22:22:26.789Z
159882024-02-06T22:22:26.789ZVerifying data now
159892024-02-06T22:22:32.412ZNo change after verify
159902024-02-06T22:22:32.412ZLoop: 73 Downstairs dump after verify (and repair):
159912024-02-06T22:22:32.429ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159922024-02-06T22:22:32.429Z 0 000-019 130 130 130 246 246 246 F F F
159932024-02-06T22:22:32.429Z 1 020-039 140 140 140 262 262 262 F F F
159942024-02-06T22:22:32.429Z 2 040-059 143 143 143 270 270 270 F F F
159952024-02-06T22:22:32.429Z 3 060-079 143 143 143 270 270 270 F F F
159962024-02-06T22:22:32.429Z 4 080-099 146 146 146 273 273 273 F F F
159972024-02-06T22:22:32.429Z 5 100-119 146 146 146 273 273 273 F F F
159982024-02-06T22:22:32.430Z 6 120-139 143 143 143 270 270 270 F F F
159992024-02-06T22:22:32.430Z 7 140-159 140 140 140 265 265 265 F F F
160002024-02-06T22:22:32.430Z 8 160-179 147 147 147 274 274 274 F F F
160012024-02-06T22:22:32.430Z 9 180-199 147 147 147 274 274 274 F F F
160022024-02-06T22:22:32.430Z 10 200-219 144 144 144 271 271 271 F F F
160032024-02-06T22:22:32.430Z 11 220-239 147 147 147 274 274 274 F F F
160042024-02-06T22:22:32.430Z 12 240-259 147 147 147 274 274 274 F F F
160052024-02-06T22:22:32.431Z 13 260-279 146 146 146 273 273 273 F F F
160062024-02-06T22:22:32.431Z 14 280-299 146 146 146 273 273 273 F F F
160072024-02-06T22:22:32.431Z 15 300-319 145 145 145 272 272 272 F F F
160082024-02-06T22:22:32.431Z 16 320-339 145 145 145 272 272 272 F F F
160092024-02-06T22:22:32.431Z 17 340-359 146 146 146 273 273 273 F F F
160102024-02-06T22:22:32.431Z 18 360-379 145 145 145 272 272 272 F F F
160112024-02-06T22:22:32.431Z 19 380-399 142 142 142 267 267 267 F F F
160122024-02-06T22:22:32.431Z 20 400-419 146 146 146 273 273 273 F F F
160132024-02-06T22:22:32.431Z 21 420-439 147 147 147 274 274 274 F F F
160142024-02-06T22:22:32.431Z 22 440-459 146 146 146 273 273 273 F F F
160152024-02-06T22:22:32.431Z 23 460-479 146 146 146 273 273 273 F F F
160162024-02-06T22:22:32.431Z 24 480-499 145 145 145 272 272 272 F F F
160172024-02-06T22:22:32.431Z 25 500-519 145 145 145 272 272 272 F F F
160182024-02-06T22:22:32.431Z 26 520-539 144 144 144 271 271 271 F F F
160192024-02-06T22:22:32.432Z 27 540-559 144 144 144 271 271 271 F F F
160202024-02-06T22:22:32.432Z 28 560-579 145 145 145 272 272 272 F F F
160212024-02-06T22:22:32.432Z 29 580-599 141 141 141 266 266 266 F F F
160222024-02-06T22:22:32.432ZMax gen: 147, Max flush: 274
160232024-02-06T22:22:32.449Z
160242024-02-06T22:22:32.449ZBegin loop 74
160252024-02-06T22:22:32.449ZDownstairs to restart: 1
160262024-02-06T22:22:32.474Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.624779451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730}
160272024-02-06T22:22:32.474Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
160282024-02-06T22:22:32.475Z debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625097898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730}
160292024-02-06T22:22:32.475Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625134193Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730}
160302024-02-06T22:22:32.475Z{"msg":"Crucible a6b5941d-dd90-46d0-9855-e080ca555bd0 has session id: 6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625154261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160312024-02-06T22:22:32.475Z{"msg":"Upstairs opts: Upstairs UUID: a6b5941d-dd90-46d0-9855-e080ca555bd0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625195695Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160322024-02-06T22:22:32.476Z{"msg":"Crucible stats registered with UUID: a6b5941d-dd90-46d0-9855-e080ca555bd0","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.62523115Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160332024-02-06T22:22:32.476Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625261056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730}
160342024-02-06T22:22:32.476Z{"msg":"a6b5941d-dd90-46d0-9855-e080ca555bd0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625299391Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160352024-02-06T22:22:32.476Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625343915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160362024-02-06T22:22:32.476Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.6253799Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160372024-02-06T22:22:32.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625425784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160382024-02-06T22:22:32.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625469558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160392024-02-06T22:22:32.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625515942Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160402024-02-06T22:22:32.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625569065Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160412024-02-06T22:22:32.478Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.62560255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160422024-02-06T22:22:32.478Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625651893Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160432024-02-06T22:22:32.478Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625684599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160442024-02-06T22:22:32.478Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625731433Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160452024-02-06T22:22:32.479Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.6257533Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160462024-02-06T22:22:32.479Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.625772297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160472024-02-06T22:22:32.479Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.626311224Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160482024-02-06T22:22:32.479Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.626349239Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160492024-02-06T22:22:32.480Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.626365257Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160502024-02-06T22:22:32.480Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.626407731Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160512024-02-06T22:22:32.480Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627017309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160522024-02-06T22:22:32.480Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627072762Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160532024-02-06T22:22:32.481Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627592431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160542024-02-06T22:22:32.481Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627628656Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160552024-02-06T22:22:32.481Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627644034Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160562024-02-06T22:22:32.481Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627710485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160572024-02-06T22:22:32.481Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627736152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160582024-02-06T22:22:32.481Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627768468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160592024-02-06T22:22:32.482Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.627791494Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160602024-02-06T22:22:32.482Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.62781963Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160612024-02-06T22:22:32.482Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628251982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160622024-02-06T22:22:32.482Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628693423Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160632024-02-06T22:22:32.482Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628735637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160642024-02-06T22:22:32.483Z{"msg":"[0]R flush_numbers[0..12]: [246, 262, 270, 270, 273, 273, 270, 265, 274, 274, 271, 274]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.62878538Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160652024-02-06T22:22:32.483Z{"msg":"[0]R generation[0..12]: [130, 140, 143, 143, 146, 146, 143, 140, 147, 147, 144, 147]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
160662024-02-06T22:22:32.483Z,"time":"2024-02-06T22:22:31.628821255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160672024-02-06T22:22:32.483Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628850121Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160682024-02-06T22:22:32.483Z{"msg":"[1]R flush_numbers[0..12]: [246, 262, 270, 270, 273, 273, 270, 265, 274, 274, 271, 274]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628881017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160692024-02-06T22:22:32.483Z{"msg":"[1]R generation[0..12]: [130, 140, 143, 143, 146, 146, 143, 140, 147, 147, 144, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628903654Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160702024-02-06T22:22:32.484Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.62893661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160712024-02-06T22:22:32.484Z{"msg":"[2]R flush_numbers[0..12]: [246, 262, 270, 270, 273, 273, 270, 265, 274, 274, 271, 274]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628963206Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160722024-02-06T22:22:32.484Z{"msg":"[2]R generation[0..12]: [130, 140, 143, 143, 146, 146, 143, 140, 147, 147, 144, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.628996731Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160732024-02-06T22:22:32.484Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629022788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160742024-02-06T22:22:32.484Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629065162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160752024-02-06T22:22:32.484Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629098558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160762024-02-06T22:22:32.484Z{"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629121545Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160772024-02-06T22:22:32.484Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629140372Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160782024-02-06T22:22:32.485Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629161529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160792024-02-06T22:22:32.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629178777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160802024-02-06T22:22:32.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629208823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160812024-02-06T22:22:32.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.6292267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160822024-02-06T22:22:32.486Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629257136Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
160832024-02-06T22:22:32.486Z,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160842024-02-06T22:22:32.486Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629281773Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160852024-02-06T22:22:32.486Z{"msg":"a6b5941d-dd90-46d0-9855-e080ca555bd0 is now active with session: 6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629308339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160862024-02-06T22:22:32.486Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629328377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730,"session_id":"6e3208ea-b3dc-42b1-9a0e-9c934bb8c9ed"}
160872024-02-06T22:22:32.487Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:31.629357173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730}
160882024-02-06T22:22:32.487ZLoading write count information from file "/tmp/test_repair_verify.data"
160892024-02-06T22:22:32.487ZRun Repair workload
160902024-02-06T22:22:32.487Z01/30 Write block 339 len 4096 data: 14
160912024-02-06T22:22:32.487Z02/30 Flush
160922024-02-06T22:22:32.487Z03/30 Write block 38 len 16384 data: 8 8 14 13
160932024-02-06T22:22:32.488Z04/30 Read block 179 len 12288
160942024-02-06T22:22:32.488Z05/30 Read block 50 len 24576
160952024-02-06T22:22:32.488Z06/30 Write block 37 len 8192 data: 7 9
160962024-02-06T22:22:32.488Z07/30 Read block 219 len 40960
160972024-02-06T22:22:32.488Z08/30 Read block 393 len 40960
160982024-02-06T22:22:32.488Z09/30 Write block 138 len 32768 data: 9 8 6 6 7 9 10 11
160992024-02-06T22:22:32.489Z10/30 Flush
161002024-02-06T22:22:32.489Z11/30 Write block 317 len 20480 data: 15 16 14 12 13
161012024-02-06T22:22:32.489Z12/30 Read block 333 len 40960
161022024-02-06T22:22:32.498Z13/30 Read block 6 len 24576
161032024-02-06T22:22:32.498Z14/30 Flush
161042024-02-06T22:22:32.498Z15/30 Write block 518 len 36864 data: 11 5 7 8 8 7 7 7 11
161052024-02-06T22:22:32.498Z16/30 Read block 389 len 24576
161062024-02-06T22:22:32.498Z17/30 Write block 26 len 28672 data: 9 9 12 10 9 6 4
161072024-02-06T22:22:32.499Z18/30 Write block 590 len 8192 data: 20 15
161082024-02-06T22:22:32.499Z19/30 Read block 135 len 4096
161092024-02-06T22:22:32.499Z20/30 Write block 142 len 36864 data: 8 10 11 12 14 15 17 19 15
161102024-02-06T22:22:32.499Z21/30 Read block 524 len 28672
161112024-02-06T22:22:32.499Z22/30 Read block 362 len 12288
161122024-02-06T22:22:32.499Z23/30 Read block 549 len 4096
161132024-02-06T22:22:32.499Z24/30 Write block 116 len 40960 data: 13 15 14 12 11 12 13 15 15 15
161142024-02-06T22:22:32.499Z25/30 Write block 572 len 40960 data: 16 15 16 13 13 14 14 14 13 9
161152024-02-06T22:22:32.500Z26/30 Read block 91 len 8192
161162024-02-06T22:22:32.500Z27/30 Read block 86 len 28672
161172024-02-06T22:22:32.500Z28/30 Write block 461 len 24576 data: 10 10 9 10 9 8
161182024-02-06T22:22:32.500Z29/30 Read block 347 len 12288
161192024-02-06T22:22:32.500Z30/30 Write block 529 len 20480 data: 14 16 17 15 18
161202024-02-06T22:22:32.500Z----------------------------------------------------------------
161212024-02-06T22:22:32.500Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
161222024-02-06T22:22:32.500ZStates: Active Active Active
161232024-02-06T22:22:32.501ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161242024-02-06T22:22:32.501Z 1 Acked 1000 Write 1 Done Done Done false
161252024-02-06T22:22:32.501Z 2 Acked 1001 Flush 0 Done Sent Done false
161262024-02-06T22:22:32.501Z 3 Acked 1002 Write 4 Done Sent Done false
161272024-02-06T22:22:32.501Z 4 Acked 1003 Read 3 Done Sent Done false
161282024-02-06T22:22:32.501Z 5 Acked 1004 Read 6 Done Sent Done false
161292024-02-06T22:22:32.501Z 6 Acked 1005 Write 2 Done Sent Done false
161302024-02-06T22:22:32.502Z 7 Acked 1006 Read 10 Done Sent Done false
161312024-02-06T22:22:32.502Z 8 Acked 1007 Read 10 Done Sent Done false
161322024-02-06T22:22:32.502Z 9 Acked 1008 Write 8 Done Sent Done false
161332024-02-06T22:22:32.502Z 10 Acked 1009 Flush 0 Done Sent Done false
161342024-02-06T22:22:32.502Z 11 Acked 1010 Write 5 Done Sent Done false
161352024-02-06T22:22:32.502Z 12 Acked 1011 Read 10 Done Sent Done false
161362024-02-06T22:22:32.502Z 13 Acked 1012 Read 6 Done Sent Done false
161372024-02-06T22:22:32.503Z 14 Acked 1013 Flush 0 Done Sent Done false
161382024-02-06T22:22:32.503Z 15 Acked 1014 Write 9 Done Sent Done false
161392024-02-06T22:22:32.503Z 16 Acked 1015 Read 6 Done Sent Done false
161402024-02-06T22:22:32.503Z 17 Acked 1016 Write 7 Done Sent Done false
161412024-02-06T22:22:32.503Z 18 Acked 1017 Write 2 Done Sent Done false
161422024-02-06T22:22:32.503Z 19 Acked 1018 Read 1 Done Sent Done false
161432024-02-06T22:22:32.504Z 20 Acked 1019 Write 9 Done Sent Done false
161442024-02-06T22:22:32.504Z 21 Acked 1020 Read 7 Done Sent Done false
161452024-02-06T22:22:32.504Z 22 Acked 1021 Read 3 Done Sent Done false
161462024-02-06T22:22:32.504Z 23 Acked 1022 Read 1 Done Sent Done false
161472024-02-06T22:22:32.504Z 24 Acked 1023 Write 10 Done Sent Done false
161482024-02-06T22:22:32.504Z 25 Acked 1024 Write 10 Done Sent Done false
161492024-02-06T22:22:32.505Z 26 Acked 1025 Read 2 Done Sent Done false
161502024-02-06T22:22:32.505Z 27 Acked 1026 Read 7 Done Sent Done false
161512024-02-06T22:22:32.505Z 28 Acked 1027 Write 6 Done Sent Done false
161522024-02-06T22:22:32.505Z 29 Acked 1028 Read 3 Done Sent Done false
161532024-02-06T22:22:32.505Z 30 Acked 1029 Write 5 Sent Sent Sent false
161542024-02-06T22:22:32.505Z STATES DS:0 DS:1 DS:2 TOTAL
161552024-02-06T22:22:32.505Z New 0 0 0 0
161562024-02-06T22:22:32.506Z Sent 1 29 1 31
161572024-02-06T22:22:32.506Z Done 29 1 29 59
161582024-02-06T22:22:32.506Z Skipped 0 0 0 0
161592024-02-06T22:22:32.506Z Error 0 0 0 0
161602024-02-06T22:22:32.506ZLast Flush: 1013 0 1013
161612024-02-06T22:22:32.506ZDownstairs last five completed:
161622024-02-06T22:22:32.506ZUpstairs last five completed: 30 29 28 27 26
161632024-02-06T22:22:32.506Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:31.661315705Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1730}
161642024-02-06T22:22:32.507ZWrote out file "/tmp/test_repair_verify.data"
161652024-02-06T22:22:32.507Z
161662024-02-06T22:22:32.507ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
161672024-02-06T22:22:32.514ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161682024-02-06T22:22:32.514Z 0 000-019 130 130 130 246 246 246 F F F
161692024-02-06T22:22:32.514Z 1 020-039 148 140 148 276 262 276 T F T <---
161702024-02-06T22:22:32.515Z 2 040-059 148 143 148 276 270 276 F F F <---
161712024-02-06T22:22:32.515Z 3 060-079 143 143 143 270 270 270 F F F
161722024-02-06T22:22:32.515Z 4 080-099 146 146 146 273 273 273 F F F
161732024-02-06T22:22:32.515Z 5 100-119 146 146 146 273 273 273 T F T <---
161742024-02-06T22:22:32.515Z 6 120-139 148 143 148 276 270 276 T F T <---
161752024-02-06T22:22:32.515Z 7 140-159 148 140 148 276 265 276 T F T <---
161762024-02-06T22:22:32.516Z 8 160-179 147 147 147 274 274 274 F F F
161772024-02-06T22:22:32.516Z 9 180-199 147 147 147 274 274 274 F F F
161782024-02-06T22:22:32.516Z 10 200-219 144 144 144 271 271 271 F F F
161792024-02-06T22:22:32.516Z 11 220-239 147 147 147 274 274 274 F F F
161802024-02-06T22:22:32.516Z 12 240-259 147 147 147 274 274 274 F F F
161812024-02-06T22:22:32.516Z 13 260-279 146 146 146 273 273 273 F F F
161822024-02-06T22:22:32.517Z 14 280-299 146 146 146 273 273 273 F F F
161832024-02-06T22:22:32.517Z 15 300-319 148 145 148 277 272 277 F F F <---
161842024-02-06T22:22:32.517Z 16 320-339 148 145 148 277 272 277 F T F <---
161852024-02-06T22:22:32.517Z 17 340-359 146 146 146 273 273 273 F F F
161862024-02-06T22:22:32.517Z 18 360-379 145 145 145 272 272 272 F F F
161872024-02-06T22:22:32.517Z 19 380-399 142 142 142 267 267 267 F F F
161882024-02-06T22:22:32.518Z 20 400-419 146 146 146 273 273 273 F F F
161892024-02-06T22:22:32.518Z 21 420-439 147 147 147 274 274 274 F F F
161902024-02-06T22:22:32.518Z 22 440-459 146 146 146 273 273 273 F F F
161912024-02-06T22:22:32.518Z 23 460-479 146 146 146 273 273 273 T F T <---
161922024-02-06T22:22:32.518Z 24 480-499 145 145 145 272 272 272 F F F
161932024-02-06T22:22:32.519Z 25 500-519 145 145 145 272 272 272 T F T <---
161942024-02-06T22:22:32.519Z 26 520-539 144 144 144 271 271 271 T F T <---
161952024-02-06T22:22:32.519Z 27 540-559 144 144 144 271 271 271 F F F
161962024-02-06T22:22:32.519Z 28 560-579 145 145 145 272 272 272 T F T <---
161972024-02-06T22:22:32.520Z 29 580-599 141 141 141 266 266 266 T F T <---
161982024-02-06T22:22:32.520ZMax gen: 148, Max flush: 277
161992024-02-06T22:22:32.520ZError: Difference in extent metadata found!
162002024-02-06T22:22:32.520ZA Difference in extent metadata is expected here
162012024-02-06T22:22:32.520ZOn loop 74
162022024-02-06T22:22:32.521Z
162032024-02-06T22:22:32.521ZVerifying data now
162042024-02-06T22:22:38.127ZNo change after verify
162052024-02-06T22:22:38.127ZLoop: 74 Downstairs dump after verify (and repair):
162062024-02-06T22:22:38.149ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162072024-02-06T22:22:38.149Z 0 000-019 130 130 130 246 246 246 F F F
162082024-02-06T22:22:38.149Z 1 020-039 149 149 149 278 278 278 F F F
162092024-02-06T22:22:38.149Z 2 040-059 148 148 148 276 276 276 F F F
162102024-02-06T22:22:38.149Z 3 060-079 143 143 143 270 270 270 F F F
162112024-02-06T22:22:38.149Z 4 080-099 146 146 146 273 273 273 F F F
162122024-02-06T22:22:38.149Z 5 100-119 149 149 149 278 278 278 F F F
162132024-02-06T22:22:38.149Z 6 120-139 149 149 149 278 278 278 F F F
162142024-02-06T22:22:38.149Z 7 140-159 149 149 149 278 278 278 F F F
162152024-02-06T22:22:38.149Z 8 160-179 147 147 147 274 274 274 F F F
162162024-02-06T22:22:38.149Z 9 180-199 147 147 147 274 274 274 F F F
162172024-02-06T22:22:38.149Z 10 200-219 144 144 144 271 271 271 F F F
162182024-02-06T22:22:38.149Z 11 220-239 147 147 147 274 274 274 F F F
162192024-02-06T22:22:38.153Z 12 240-259 147 147 147 274 274 274 F F F
162202024-02-06T22:22:38.153Z 13 260-279 146 146 146 273 273 273 F F F
162212024-02-06T22:22:38.153Z 14 280-299 146 146 146 273 273 273 F F F
162222024-02-06T22:22:38.153Z 15 300-319 148 148 148 277 277 277 F F F
162232024-02-06T22:22:38.153Z 16 320-339 148 148 148 277 277 277 F F F
162242024-02-06T22:22:38.153Z 17 340-359 146 146 146 273 273 273 F F F
162252024-02-06T22:22:38.153Z 18 360-379 145 145 145 272 272 272 F F F
162262024-02-06T22:22:38.153Z 19 380-399 142 142 142 267 267 267 F F F
162272024-02-06T22:22:38.153Z 20 400-419 146 146 146 273 273 273 F F F
162282024-02-06T22:22:38.153Z 21 420-439 147 147 147 274 274 274 F F F
162292024-02-06T22:22:38.153Z 22 440-459 146 146 146 273 273 273 F F F
162302024-02-06T22:22:38.153Z 23 460-479 149 149 149 278 278 278 F F F
162312024-02-06T22:22:38.153Z 24 480-499 145 145 145 272 272 272 F F F
162322024-02-06T22:22:38.153Z 25 500-519 149 149 149 278 278 278 F F F
162332024-02-06T22:22:38.153Z 26 520-539 149 149 149 278 278 278 F F F
162342024-02-06T22:22:38.153Z 27 540-559 144 144 144 271 271 271 F F F
162352024-02-06T22:22:38.154Z 28 560-579 149 149 149 278 278 278 F F F
162362024-02-06T22:22:38.154Z 29 580-599 149 149 149 278 278 278 F F F
162372024-02-06T22:22:38.154ZMax gen: 149, Max flush: 278
162382024-02-06T22:22:38.154Z
162392024-02-06T22:22:38.154ZBegin loop 75
162402024-02-06T22:22:38.154ZDownstairs to restart: 2
162412024-02-06T22:22:38.158Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.338664017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738}
162422024-02-06T22:22:38.158Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.338960477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738}
162432024-02-06T22:22:38.158Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
162442024-02-06T22:22:38.158Z,"time":"2024-02-06T22:22:37.339060033Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738}
162452024-02-06T22:22:38.158Z{"msg":"Crucible d9a9a8db-54b6-4a20-8126-199c75868a2b has session id: c610fdde-935f-41ba-99bf-38a0ec9f0cd7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339091839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162462024-02-06T22:22:38.159Z{"msg":"Upstairs opts: Upstairs UUID: d9a9a8db-54b6-4a20-8126-199c75868a2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339137243Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162472024-02-06T22:22:38.159Z{"msg":"Crucible stats registered with UUID: d9a9a8db-54b6-4a20-8126-199c75868a2b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339174088Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162482024-02-06T22:22:38.159Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339190066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738}
162492024-02-06T22:22:38.159Z{"msg":"d9a9a8db-54b6-4a20-8126-199c75868a2b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339217702Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162502024-02-06T22:22:38.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.33923103Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162512024-02-06T22:22:38.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339258077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162522024-02-06T22:22:38.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339272535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162532024-02-06T22:22:38.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339287303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162542024-02-06T22:22:38.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339301041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162552024-02-06T22:22:38.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339318478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162562024-02-06T22:22:38.159Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339351184Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162572024-02-06T22:22:38.159Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339376601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162582024-02-06T22:22:38.159Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339469378Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162592024-02-06T22:22:38.159Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:37.339521631Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162602024-02-06T22:22:38.159Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:37.339555886Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162612024-02-06T22:22:38.159Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:37.339588262Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162622024-02-06T22:22:38.159Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339624137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162632024-02-06T22:22:38.159Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339650014Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162642024-02-06T22:22:38.159Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339691308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162652024-02-06T22:22:38.159Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339718555Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162662024-02-06T22:22:38.159Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.339758259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162672024-02-06T22:22:38.159Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.340244483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162682024-02-06T22:22:38.159Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.340291687Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162692024-02-06T22:22:38.159Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.340322543Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162702024-02-06T22:22:38.160Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.340360208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162712024-02-06T22:22:38.160Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.340877628Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162722024-02-06T22:22:38.163Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.341027807Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162732024-02-06T22:22:38.163Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.341437412Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162742024-02-06T22:22:38.163Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.341478387Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162752024-02-06T22:22:38.163Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.341550697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162762024-02-06T22:22:38.163Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.341582672Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162772024-02-06T22:22:38.163Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.34159873Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162782024-02-06T22:22:38.163Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:37.341619667Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162792024-02-06T22:22:39.158Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.341116824Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162802024-02-06T22:22:39.163Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.341382548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162812024-02-06T22:22:39.163Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.342332639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162822024-02-06T22:22:39.163Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.342370744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162832024-02-06T22:22:39.163Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.342815164Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162842024-02-06T22:22:39.163Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343325715Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162852024-02-06T22:22:39.163Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343368339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162862024-02-06T22:22:39.163Z{"msg":"[0]R flush_numbers[0..12]: [246, 278, 276, 270, 273, 278, 278, 278, 274, 274, 271, 274]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343387777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162872024-02-06T22:22:39.163Z{"msg":"[0]R generation[0..12]: [130, 149, 148, 143, 146, 149, 149, 149, 147, 147, 144, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343428821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162882024-02-06T22:22:39.163Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343459797Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162892024-02-06T22:22:39.163Z{"msg":"[1]R flush_numbers[0..12]: [246, 278, 276, 270, 273, 278, 278, 278, 274, 274, 271, 274]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343495132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162902024-02-06T22:22:39.163Z{"msg":"[1]R generation[0..12]: [130, 149, 148, 143, 146, 149, 149, 149, 147, 147, 144, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.34351537Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162912024-02-06T22:22:39.163ZWait for a query_work_queue command to finish before sending IO
162922024-02-06T22:22:39.163Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343559444Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162932024-02-06T22:22:39.163Z{"msg":"[2]R flush_numbers[0..12]: [246, 278, 276, 270, 273, 278, 278, 278, 274, 274, 271, 274]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.34358798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162942024-02-06T22:22:39.164Z{"msg":"[2]R generation[0..12]: [130, 149, 148, 143, 146, 149, 149, 149, 147, 147, 144, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343623885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162952024-02-06T22:22:39.164Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343644032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162962024-02-06T22:22:39.164Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343680977Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162972024-02-06T22:22:39.164Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343702404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162982024-02-06T22:22:39.164Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.34373874Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
162992024-02-06T22:22:39.164Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343757267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163002024-02-06T22:22:39.164Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343795362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163012024-02-06T22:22:39.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343814279Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163022024-02-06T22:22:39.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343850784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163032024-02-06T22:22:39.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343869252Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163042024-02-06T22:22:39.164Z"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163052024-02-06T22:22:39.164Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343907267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163062024-02-06T22:22:39.164Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343925924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163072024-02-06T22:22:39.164Z{"msg":"d9a9a8db-54b6-4a20-8126-199c75868a2b is now active with session: c610fdde-935f-41ba-99bf-38a0ec9f0cd7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343960459Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163082024-02-06T22:22:39.164Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.343977567Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738,"session_id":"c610fdde-935f-41ba-99bf-38a0ec9f0cd7"}
163092024-02-06T22:22:39.164Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:38.344008263Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738}
163102024-02-06T22:22:39.164ZLoading write count information from file "/tmp/test_repair_verify.data"
163112024-02-06T22:22:39.164ZRun Repair workload
163122024-02-06T22:22:39.164Z01/30 Flush
163132024-02-06T22:22:39.168Z02/30 Write block 265 len 16384 data: 19 20 19 17
163142024-02-06T22:22:39.168Z03/30 Write block 47 len 8192 data: 10 11
163152024-02-06T22:22:39.168Z04/30 Read block 165 len 24576
163162024-02-06T22:22:39.168Z05/30 Write block 30 len 40960 data: 10 7 5 6 6 5 6 8 10 9
163172024-02-06T22:22:39.168Z06/30 Write block 14 len 20480 data: 9 10 9 8 9
163182024-02-06T22:22:39.168Z07/30 Read block 385 len 16384
163192024-02-06T22:22:39.168Z08/30 Read block 395 len 16384
163202024-02-06T22:22:39.172Z09/30 Write block 568 len 28672 data: 15 13 13 17 17 16 17
163212024-02-06T22:22:39.172Z10/30 Write block 296 len 36864 data: 13 9 9 10 13 14 15 15 13
163222024-02-06T22:22:39.172Z11/30 Read block 98 len 28672
163232024-02-06T22:22:39.175Z12/30 Write block 142 len 36864 data: 9 11 12 13 15 16 18 20 16
163242024-02-06T22:22:39.175Z13/30 Read block 409 len 28672
163252024-02-06T22:22:39.175Z14/30 Read block 552 len 16384
163262024-02-06T22:22:39.175Z15/30 Flush
163272024-02-06T22:22:39.178Z16/30 Flush
163282024-02-06T22:22:39.178Z17/30 Write block 444 len 8192 data: 8 8
163292024-02-06T22:22:39.178Z18/30 Read block 327 len 20480
163302024-02-06T22:22:39.181Z19/30 Read block 349 len 40960
163312024-02-06T22:22:39.181Z20/30 Flush
163322024-02-06T22:22:39.184Z21/30 Write block 339 len 28672 data: 15 15 15 19 14 15 12
163332024-02-06T22:22:39.184Z22/30 Read block 433 len 4096
163342024-02-06T22:22:39.187Z23/30 Write block 428 len 40960 data: 12 11 10 6 6 7 7 8 11 13
163352024-02-06T22:22:39.187Z24/30 Write block 411 len 16384 data: 15 12 12 11
163362024-02-06T22:22:39.187Z25/30 Read block 44 len 36864
163372024-02-06T22:22:39.187Z26/30 Write block 193 len 16384 data: 10 13 11 14
163382024-02-06T22:22:39.190Z27/30 Write block 222 len 40960 data: 16 16 16 13 13 9 9 9 9 12
163392024-02-06T22:22:39.190Z28/30 Write block 412 len 4096 data: 13
163402024-02-06T22:22:39.190Z29/30 Read block 177 len 36864
163412024-02-06T22:22:39.193Z30/30 Write block 581 len 40960 data: 10 7 8 11 13 13 14 14 17 21
163422024-02-06T22:22:39.194Z----------------------------------------------------------------
163432024-02-06T22:22:39.194Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:29
163442024-02-06T22:22:39.194ZStates: Active Active Active
163452024-02-06T22:22:39.194ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
163462024-02-06T22:22:39.194Z 2 Acked 1001 Write 4 Done Done Done false
163472024-02-06T22:22:39.194Z 3 Acked 1002 Write 2 Done Done Done false
163482024-02-06T22:22:39.194Z 4 Acked 1003 Read 6 Done Done Done false
163492024-02-06T22:22:39.194Z 5 Acked 1004 Write 10 Done Done Done false
163502024-02-06T22:22:39.194Z 6 Acked 1005 Write 5 Done Done Done false
163512024-02-06T22:22:39.194Z 7 Acked 1006 Read 4 Done Done Done false
163522024-02-06T22:22:39.194Z 8 Acked 1007 Read 4 Done Done Done false
163532024-02-06T22:22:39.194Z 9 Acked 1008 Write 7 Done Done Done false
163542024-02-06T22:22:39.194Z 10 Acked 1009 Write 9 Done Done Done false
163552024-02-06T22:22:39.194Z 11 Acked 1010 Read 7 Done Done Done false
163562024-02-06T22:22:39.194Z 12 Acked 1011 Write 9 Done Done Done false
163572024-02-06T22:22:39.194Z 13 Acked 1012 Read 7 Done Done Sent false
163582024-02-06T22:22:39.194Z 14 Acked 1013 Read 4 Done Done Sent false
163592024-02-06T22:22:39.194Z 15 Acked 1014 Flush 0 Done Done Sent false
163602024-02-06T22:22:39.194Z 16 Acked 1015 Flush 0 Done Done Sent false
163612024-02-06T22:22:39.194Z 17 Acked 1016 Write 2 Done Done Sent false
163622024-02-06T22:22:39.194Z 18 Acked 1017 Read 5 Done Done Sent false
163632024-02-06T22:22:39.194Z 19 Acked 1018 Read 10 Done Done Sent false
163642024-02-06T22:22:39.194Z 20 Acked 1019 Flush 0 Done Done Sent false
163652024-02-06T22:22:39.194Z 21 Acked 1020 Write 7 Done Done Sent false
163662024-02-06T22:22:39.195Z 22 Acked 1021 Read 1 Done Done Sent false
163672024-02-06T22:22:39.195Z 23 Acked 1022 Write 10 Done Done Sent false
163682024-02-06T22:22:39.195Z 24 Acked 1023 Write 4 Done Done Sent false
163692024-02-06T22:22:39.195Z 25 Acked 1024 Read 9 Done Done Sent false
163702024-02-06T22:22:39.195Z 26 Acked 1025 Write 4 Done Done Sent false
163712024-02-06T22:22:39.195Z 27 Acked 1026 Write 10 Done Done Sent false
163722024-02-06T22:22:39.195Z 28 Acked 1027 Write 1 Done Done Sent false
163732024-02-06T22:22:39.195Z 29 Acked 1028 Read 9 Done Done Sent false
163742024-02-06T22:22:39.195Z 30 Acked 1029 Write 10 Sent Sent Sent false
163752024-02-06T22:22:39.195Z STATES DS:0 DS:1 DS:2 TOTAL
163762024-02-06T22:22:39.195Z New 0 0 0 0
163772024-02-06T22:22:39.195Z Sent 1 1 18 20
163782024-02-06T22:22:39.195Z Done 28 28 11 67
163792024-02-06T22:22:39.195Z Skipped 0 0 0 0
163802024-02-06T22:22:39.195Z Error 0 0 0 0
163812024-02-06T22:22:39.195ZLast Flush: 1019 1019 1000
163822024-02-06T22:22:39.195ZDownstairs last five completed: 1000
163832024-02-06T22:22:39.195ZUpstairs last five completed: 30 29 28 27 26
163842024-02-06T22:22:39.195Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:38.375237984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1738}
163852024-02-06T22:22:39.195ZWrote out file "/tmp/test_repair_verify.data"
163862024-02-06T22:22:39.198Z
163872024-02-06T22:22:39.201ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163882024-02-06T22:22:39.219ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163892024-02-06T22:22:39.219Z 0 000-019 150 150 130 280 280 246 F F T <---
163902024-02-06T22:22:39.219Z 1 020-039 150 150 149 280 280 278 F F T <---
163912024-02-06T22:22:39.219Z 2 040-059 150 150 148 280 280 276 F F T <---
163922024-02-06T22:22:39.219Z 3 060-079 143 143 143 270 270 270 F F F
163932024-02-06T22:22:39.219Z 4 080-099 146 146 146 273 273 273 F F F
163942024-02-06T22:22:39.219Z 5 100-119 149 149 149 278 278 278 F F F
163952024-02-06T22:22:39.220Z 6 120-139 149 149 149 278 278 278 F F F
163962024-02-06T22:22:39.220Z 7 140-159 150 150 149 280 280 278 F F T <---
163972024-02-06T22:22:39.220Z 8 160-179 147 147 147 274 274 274 F F F
163982024-02-06T22:22:39.220Z 9 180-199 147 147 147 274 274 274 T T F <---
163992024-02-06T22:22:39.223Z 10 200-219 144 144 144 271 271 271 F F F
164002024-02-06T22:22:39.223Z 11 220-239 147 147 147 274 274 274 T T F <---
164012024-02-06T22:22:39.223Z 12 240-259 147 147 147 274 274 274 F F F
164022024-02-06T22:22:39.223Z 13 260-279 150 150 146 280 280 273 F F T <---
164032024-02-06T22:22:39.223Z 14 280-299 150 150 146 280 280 273 F F T <---
164042024-02-06T22:22:39.223Z 15 300-319 150 150 148 280 280 277 F F T <---
164052024-02-06T22:22:39.223Z 16 320-339 148 148 148 277 277 277 T T F <---
164062024-02-06T22:22:39.223Z 17 340-359 146 146 146 273 273 273 T T F <---
164072024-02-06T22:22:39.223Z 18 360-379 145 145 145 272 272 272 F F F
164082024-02-06T22:22:39.223Z 19 380-399 142 142 142 267 267 267 F F F
164092024-02-06T22:22:39.223Z 20 400-419 146 146 146 273 273 273 T T F <---
164102024-02-06T22:22:39.224Z 21 420-439 147 147 147 274 274 274 T T F <---
164112024-02-06T22:22:39.224Z 22 440-459 150 150 146 282 282 273 F F F <---
164122024-02-06T22:22:39.224Z 23 460-479 149 149 149 278 278 278 F F F
164132024-02-06T22:22:39.224Z 24 480-499 145 145 145 272 272 272 F F F
164142024-02-06T22:22:39.224Z 25 500-519 149 149 149 278 278 278 F F F
164152024-02-06T22:22:39.224Z 26 520-539 149 149 149 278 278 278 F F F
164162024-02-06T22:22:39.224Z 27 540-559 144 144 144 271 271 271 F F F
164172024-02-06T22:22:39.224Z 28 560-579 150 150 149 280 280 278 F F T <---
164182024-02-06T22:22:39.224Z 29 580-599 149 149 149 278 278 278 T T F <---
164192024-02-06T22:22:39.224ZMax gen: 150, Max flush: 282
164202024-02-06T22:22:39.224ZError: Difference in extent metadata found!
164212024-02-06T22:22:39.224ZA Difference in extent metadata is expected here
164222024-02-06T22:22:39.224ZOn loop 75
164232024-02-06T22:22:39.224Z
164242024-02-06T22:22:39.227ZVerifying data now
164252024-02-06T22:22:44.992ZNo change after verify
164262024-02-06T22:22:44.992ZLoop: 75 Downstairs dump after verify (and repair):
164272024-02-06T22:22:45.007ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164282024-02-06T22:22:45.007Z 0 000-019 150 150 150 280 280 280 F F F
164292024-02-06T22:22:45.007Z 1 020-039 150 150 150 280 280 280 F F F
164302024-02-06T22:22:45.007Z 2 040-059 150 150 150 280 280 280 F F F
164312024-02-06T22:22:45.007Z 3 060-079 143 143 143 270 270 270 F F F
164322024-02-06T22:22:45.007Z 4 080-099 146 146 146 273 273 273 F F F
164332024-02-06T22:22:45.008Z 5 100-119 149 149 149 278 278 278 F F F
164342024-02-06T22:22:45.008Z 6 120-139 149 149 149 278 278 278 F F F
164352024-02-06T22:22:45.008Z 7 140-159 150 150 150 280 280 280 F F F
164362024-02-06T22:22:45.008Z 8 160-179 147 147 147 274 274 274 F F F
164372024-02-06T22:22:45.008Z 9 180-199 151 151 151 283 283 283 F F F
164382024-02-06T22:22:45.008Z 10 200-219 144 144 144 271 271 271 F F F
164392024-02-06T22:22:45.015Z 11 220-239 151 151 151 283 283 283 F F F
164402024-02-06T22:22:45.015Z 12 240-259 147 147 147 274 274 274 F F F
164412024-02-06T22:22:45.015Z 13 260-279 150 150 150 280 280 280 F F F
164422024-02-06T22:22:45.015Z 14 280-299 150 150 150 280 280 280 F F F
164432024-02-06T22:22:45.016Z 15 300-319 150 150 150 280 280 280 F F F
164442024-02-06T22:22:45.016Z 16 320-339 151 151 151 283 283 283 F F F
164452024-02-06T22:22:45.016Z 17 340-359 151 151 151 283 283 283 F F F
164462024-02-06T22:22:45.016Z 18 360-379 145 145 145 272 272 272 F F F
164472024-02-06T22:22:45.016Z 19 380-399 142 142 142 267 267 267 F F F
164482024-02-06T22:22:45.016Z 20 400-419 151 151 151 283 283 283 F F F
164492024-02-06T22:22:45.016Z 21 420-439 151 151 151 283 283 283 F F F
164502024-02-06T22:22:45.016Z 22 440-459 150 150 150 282 282 282 F F F
164512024-02-06T22:22:45.016Z 23 460-479 149 149 149 278 278 278 F F F
164522024-02-06T22:22:45.016Z 24 480-499 145 145 145 272 272 272 F F F
164532024-02-06T22:22:45.016Z 25 500-519 149 149 149 278 278 278 F F F
164542024-02-06T22:22:45.016Z 26 520-539 149 149 149 278 278 278 F F F
164552024-02-06T22:22:45.016Z 27 540-559 144 144 144 271 271 271 F F F
164562024-02-06T22:22:45.016Z 28 560-579 150 150 150 280 280 280 F F F
164572024-02-06T22:22:45.016Z 29 580-599 151 151 151 283 283 283 F F F
164582024-02-06T22:22:45.016ZMax gen: 151, Max flush: 283
164592024-02-06T22:22:45.016Z
164602024-02-06T22:22:45.016ZBegin loop 76
164612024-02-06T22:22:45.016ZDownstairs to restart: 0
164622024-02-06T22:22:45.024Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205104228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746}
164632024-02-06T22:22:45.024Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205369612Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746}
164642024-02-06T22:22:45.042Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205403838Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746}
164652024-02-06T22:22:45.043Z{"msg":"Crucible runtime is spawned
164662024-02-06T22:22:45.043ZCrucible 6c1c07ca-fa7a-49ac-a001-face6f207b8e has session id: 4fcf4623-53c7-49e6-9344-fa055fc71629","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205442643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164672024-02-06T22:22:45.043Z{"msg":"Upstairs opts: Upstairs UUID: 6c1c07ca-fa7a-49ac-a001-face6f207b8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205489286Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164682024-02-06T22:22:45.043Z{"msg":"Crucible stats registered with UUID: 6c1c07ca-fa7a-49ac-a001-face6f207b8e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205549198Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164692024-02-06T22:22:45.043Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205573165Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746}
164702024-02-06T22:22:45.043Z{"msg":"6c1c07ca-fa7a-49ac-a001-face6f207b8e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205669672Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164712024-02-06T22:22:45.043Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205697538Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164722024-02-06T22:22:45.043Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205713496Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164732024-02-06T22:22:45.043Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205728404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164742024-02-06T22:22:45.043Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205742552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164752024-02-06T22:22:45.043Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205772298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164762024-02-06T22:22:45.043Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205815522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164772024-02-06T22:22:45.043Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.20583456Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164782024-02-06T22:22:45.043Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205883373Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164792024-02-06T22:22:45.043Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.205911839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164802024-02-06T22:22:45.043Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.206000997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164812024-02-06T22:22:45.043Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.206030913Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164822024-02-06T22:22:45.043Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.2060568Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164832024-02-06T22:22:45.043Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.206784911Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164842024-02-06T22:22:45.043Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.206833035Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164852024-02-06T22:22:45.043Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.206862821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164862024-02-06T22:22:45.043Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.206927932Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164872024-02-06T22:22:45.043Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.207350125Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164882024-02-06T22:22:45.044Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.207477208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164892024-02-06T22:22:45.044Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.207670192Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164902024-02-06T22:22:45.044Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.207705067Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164912024-02-06T22:22:45.044Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.207879373Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164922024-02-06T22:22:45.044Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.207940415Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164932024-02-06T22:22:45.044Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.207992858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164942024-02-06T22:22:45.044Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208024214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164952024-02-06T22:22:45.044Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208057539Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164962024-02-06T22:22:45.044Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208078857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164972024-02-06T22:22:45.044Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208258322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164982024-02-06T22:22:45.044Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208713881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
164992024-02-06T22:22:45.044Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208742317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165002024-02-06T22:22:45.044Z{"msg":"[0]R flush_numbers[0..12]: [280, 280, 280, 270, 273, 278, 278, 280, 274, 283, 271, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208764784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165012024-02-06T22:22:45.044Z{"msg":"[0]R generation[0..12]: [150, 150, 150, 143, 146, 149, 149, 150, 147, 151, 144, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208796869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165022024-02-06T22:22:45.044Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208818417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165032024-02-06T22:22:45.044Z{"msg":"[1]R flush_numbers[0..12]: [280, 280, 280, 270, 273, 278, 278, 280, 274, 283, 271, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208848533Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165042024-02-06T22:22:45.044Z{"msg":"[1]R generation[0..12]: [150, 150, 150, 143, 146, 149, 149, 150, 147, 151, 144, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.20886952Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165052024-02-06T22:22:45.044Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
165062024-02-06T22:22:45.044Z,"time":"2024-02-06T22:22:44.208899866Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165072024-02-06T22:22:45.044Z{"msg":"[2]R flush_numbers[0..12]: [280, 280, 280, 270, 273, 278, 278, 280, 274, 283, 271, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208919373Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165082024-02-06T22:22:45.044Z{"msg":"[2]R generation[0..12]: [150, 150, 150, 143, 146, 149, 149, 150, 147, 151, 144, 151]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208951548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165092024-02-06T22:22:45.044Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.208972646Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165102024-02-06T22:22:45.044Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209000852Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165112024-02-06T22:22:45.045Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.20901592Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165122024-02-06T22:22:45.045Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209046966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165132024-02-06T22:22:45.045Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209062624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165142024-02-06T22:22:45.045Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209093849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165152024-02-06T22:22:45.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209107838Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165162024-02-06T22:22:45.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209138553Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165172024-02-06T22:22:45.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209156341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165182024-02-06T22:22:45.045Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209185807Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165192024-02-06T22:22:45.045Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209199975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165202024-02-06T22:22:45.045Z{"msg":"6c1c07ca-fa7a-49ac-a001-face6f207b8e is now active with session: 4fcf4623-53c7-49e6-9344-fa055fc71629","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209221222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165212024-02-06T22:22:45.045Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209246759Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165222024-02-06T22:22:45.045Zip-10-150-1-27.us-west-2.compute.internal","pid":1746,"session_id":"4fcf4623-53c7-49e6-9344-fa055fc71629"}
165232024-02-06T22:22:45.045Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:44.209263876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746}
165242024-02-06T22:22:45.045ZLoading write count information from file "/tmp/test_repair_verify.data"
165252024-02-06T22:22:45.045ZRun Repair workload
165262024-02-06T22:22:45.045Z01/30 Write block 403 len 28672 data: 12 11 13 13 11 10 13
165272024-02-06T22:22:45.045Z02/30 Write block 398 len 28672 data: 8 9 10 12 13 13 12
165282024-02-06T22:22:45.057Z03/30 Read block 155 len 24576
165292024-02-06T22:22:45.057Z04/30 Read block 295 len 36864
165302024-02-06T22:22:45.057Z05/30 Write block 149 len 28672 data: 21 17 13 14 16 15 19
165312024-02-06T22:22:45.057Z06/30 Write block 113 len 32768 data: 13 15 15 14 16 15 13 12
165322024-02-06T22:22:45.057Z07/30 Write block 12 len 40960 data: 8 7 10 11 10 9 10 8 7 7
165332024-02-06T22:22:45.057Z08/30 Read block 352 len 20480
165342024-02-06T22:22:45.057Z09/30 Read block 525 len 32768
165352024-02-06T22:22:45.057Z10/30 Flush
165362024-02-06T22:22:45.057Z11/30 Write block 65 len 40960 data: 8 9 8 9 9 9 9 10 10 13
165372024-02-06T22:22:45.057Z12/30 Flush
165382024-02-06T22:22:45.057Z13/30 Write block 555 len 4096 data: 10
165392024-02-06T22:22:45.057Z14/30 Read block 514 len 4096
165402024-02-06T22:22:45.057Z15/30 Read block 164 len 20480
165412024-02-06T22:22:45.057Z16/30 Read block 277 len 36864
165422024-02-06T22:22:45.057Z17/30 Read block 164 len 4096
165432024-02-06T22:22:45.057Z18/30 Read block 290 len 4096
165442024-02-06T22:22:45.057Z19/30 Write block 541 len 40960 data: 9 9 10 12 10 11 10 9 8 7
165452024-02-06T22:22:45.058Z20/30 Read block 549 len 20480
165462024-02-06T22:22:45.058Z21/30 Write block 321 len 12288 data: 14 16 13
165472024-02-06T22:22:45.058Z22/30 Read block 5 len 28672
165482024-02-06T22:22:45.058Z23/30 Write block 222 len 32768 data: 17 17 17 14 14 10 10 10
165492024-02-06T22:22:45.058Z24/30 Read block 446 len 8192
165502024-02-06T22:22:45.058Z25/30 Write block 222 len 8192 data: 18 18
165512024-02-06T22:22:45.058Z26/30 Read block 57 len 12288
165522024-02-06T22:22:45.058Z27/30 Read block 420 len 4096
165532024-02-06T22:22:45.064Z28/30 Write block 546 len 40960 data: 12 11 10 9 8 10 11 12 11 11
165542024-02-06T22:22:45.064Z29/30 Write block 407 len 8192 data: 12 11
165552024-02-06T22:22:45.064Z30/30 Write block 167 len 32768 data: 14 17 18 18 13 14 16 15
165562024-02-06T22:22:45.064Z----------------------------------------------------------------
165572024-02-06T22:22:45.064Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
165582024-02-06T22:22:45.065ZStates: Active Active Active
165592024-02-06T22:22:45.065ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165602024-02-06T22:22:45.065Z 1 Acked 1000 Write 7 Done Done Done false
165612024-02-06T22:22:45.065Z 2 Acked 1001 Write 7 Done Done Done false
165622024-02-06T22:22:45.065Z 3 Acked 1002 Read 6 Done Done Done false
165632024-02-06T22:22:45.065Z 4 Acked 1003 Read 9 Done Done Done false
165642024-02-06T22:22:45.065Z 5 Acked 1004 Write 7 Done Done Done false
165652024-02-06T22:22:45.065Z 6 Acked 1005 Write 8 Sent Done Done false
165662024-02-06T22:22:45.065Z 7 Acked 1006 Write 10 Sent Done Done false
165672024-02-06T22:22:45.065Z 8 Acked 1007 Read 5 Sent Done Done false
165682024-02-06T22:22:45.065Z 9 Acked 1008 Read 8 Sent Done Done false
165692024-02-06T22:22:45.065Z 10 Acked 1009 Flush 0 Sent Done Done false
165702024-02-06T22:22:45.065Z 11 Acked 1010 Write 10 Sent Done Done false
165712024-02-06T22:22:45.065Z 12 Acked 1011 Flush 0 Sent Done Done false
165722024-02-06T22:22:45.065Z 13 Acked 1012 Write 1 Sent Done Done false
165732024-02-06T22:22:45.065Z 14 Acked 1013 Read 1 Sent Done Done false
165742024-02-06T22:22:45.065Z 15 Acked 1014 Read 5 Sent Done Done false
165752024-02-06T22:22:45.065Z 16 Acked 1015 Read 9 Sent Done Done false
165762024-02-06T22:22:45.065Z 17 Acked 1016 Read 1 Sent Done Done false
165772024-02-06T22:22:45.065Z 18 Acked 1017 Read 1 Sent Done Done false
165782024-02-06T22:22:45.066Z 19 Acked 1018 Write 10 Sent Done Done false
165792024-02-06T22:22:45.066Z 20 Acked 1019 Read 5 Sent Done Done false
165802024-02-06T22:22:45.066Z 21 Acked 1020 Write 3 Sent Done Done false
165812024-02-06T22:22:45.066Z 22 Acked 1021 Read 7 Sent Done Done false
165822024-02-06T22:22:45.066Z 23 Acked 1022 Write 8 Sent Done Done false
165832024-02-06T22:22:45.066Z 24 Acked 1023 Read 2 Sent Done Done false
165842024-02-06T22:22:45.066Z 25 Acked 1024 Write 2 Sent Done Done false
165852024-02-06T22:22:45.066Z 26 Acked 1025 Read 3 Sent Done Done false
165862024-02-06T22:22:45.066Z 27 Acked 1026 Read 1 Sent Done Done false
165872024-02-06T22:22:45.066Z 28 Acked 1027 Write 10 Sent Done Done false
165882024-02-06T22:22:45.066Z 29 Acked 1028 Write 2 Sent Done Done false
165892024-02-06T22:22:45.066Z 30 Acked 1029 Write 8 Sent Sent Sent false
165902024-02-06T22:22:45.066Z STATES DS:0 DS:1 DS:2 TOTAL
165912024-02-06T22:22:45.066Z New 0 0 0 0
165922024-02-06T22:22:45.066Z Sent 25 1 1 27
165932024-02-06T22:22:45.066Z Done 5 29 29 63
165942024-02-06T22:22:45.067Z Skipped 0 0 0 0
165952024-02-06T22:22:45.067Z Error 0 0 0 0
165962024-02-06T22:22:45.067ZLast Flush: 0 1011 1011
165972024-02-06T22:22:45.067ZDownstairs last five completed:
165982024-02-06T22:22:45.067ZUpstairs last five completed: 30 29 28 27 26
165992024-02-06T22:22:45.067Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:44.240756471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1746}
166002024-02-06T22:22:45.067ZWrote out file "/tmp/test_repair_verify.data"
166012024-02-06T22:22:45.067Z
166022024-02-06T22:22:45.074ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166032024-02-06T22:22:45.087ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166042024-02-06T22:22:45.087Z 0 000-019 150 152 152 280 284 284 F F F <---
166052024-02-06T22:22:45.087Z 1 020-039 150 152 152 280 284 284 F F F <---
166062024-02-06T22:22:45.087Z 2 040-059 150 150 150 280 280 280 F F F
166072024-02-06T22:22:45.087Z 3 060-079 143 152 152 270 285 285 F F F <---
166082024-02-06T22:22:45.087Z 4 080-099 146 146 146 273 273 273 F F F
166092024-02-06T22:22:45.093Z 5 100-119 149 152 152 278 284 284 F F F <---
166102024-02-06T22:22:45.093Z 6 120-139 149 152 152 278 284 284 F F F <---
166112024-02-06T22:22:45.093Z 7 140-159 150 152 152 280 284 284 T F F <---
166122024-02-06T22:22:45.093Z 8 160-179 147 147 147 274 274 274 F T T <---
166132024-02-06T22:22:45.093Z 9 180-199 151 151 151 283 283 283 F F F
166142024-02-06T22:22:45.093Z 10 200-219 144 144 144 271 271 271 F F F
166152024-02-06T22:22:45.093Z 11 220-239 151 151 151 283 283 283 F T T <---
166162024-02-06T22:22:45.093Z 12 240-259 147 147 147 274 274 274 F F F
166172024-02-06T22:22:45.093Z 13 260-279 150 150 150 280 280 280 F F F
166182024-02-06T22:22:45.093Z 14 280-299 150 150 150 280 280 280 F F F
166192024-02-06T22:22:45.093Z 15 300-319 150 150 150 280 280 280 F F F
166202024-02-06T22:22:45.093Z 16 320-339 151 151 151 283 283 283 F T T <---
166212024-02-06T22:22:45.094Z 17 340-359 151 151 151 283 283 283 F F F
166222024-02-06T22:22:45.094Z 18 360-379 145 145 145 272 272 272 F F F
166232024-02-06T22:22:45.094Z 19 380-399 142 152 152 267 284 284 T F F <---
166242024-02-06T22:22:45.094Z 20 400-419 151 152 152 283 284 284 T T T <---
166252024-02-06T22:22:45.094Z 21 420-439 151 151 151 283 283 283 F F F
166262024-02-06T22:22:45.094Z 22 440-459 150 150 150 282 282 282 F F F
166272024-02-06T22:22:45.094Z 23 460-479 149 149 149 278 278 278 F F F
166282024-02-06T22:22:45.094Z 24 480-499 145 145 145 272 272 272 F F F
166292024-02-06T22:22:45.094Z 25 500-519 149 149 149 278 278 278 F F F
166302024-02-06T22:22:45.094Z 26 520-539 149 149 149 278 278 278 F F F
166312024-02-06T22:22:45.094Z 27 540-559 144 144 144 271 271 271 F T T <---
166322024-02-06T22:22:45.094Z 28 560-579 150 150 150 280 280 280 F F F
166332024-02-06T22:22:45.094Z 29 580-599 151 151 151 283 283 283 F F F
166342024-02-06T22:22:45.094ZMax gen: 152, Max flush: 285
166352024-02-06T22:22:45.094ZError: Difference in extent metadata found!
166362024-02-06T22:22:45.094ZA Difference in extent metadata is expected here
166372024-02-06T22:22:45.094ZOn loop 76
166382024-02-06T22:22:45.094Z
166392024-02-06T22:22:45.094ZVerifying data now
166402024-02-06T22:22:50.686ZNo change after verify
166412024-02-06T22:22:50.686ZLoop: 76 Downstairs dump after verify (and repair):
166422024-02-06T22:22:50.699ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166432024-02-06T22:22:50.700Z 0 000-019 152 152 152 284 284 284 F F F
166442024-02-06T22:22:50.700Z 1 020-039 152 152 152 284 284 284 F F F
166452024-02-06T22:22:50.700Z 2 040-059 150 150 150 280 280 280 F F F
166462024-02-06T22:22:50.700Z 3 060-079 152 152 152 285 285 285 F F F
166472024-02-06T22:22:50.700Z 4 080-099 146 146 146 273 273 273 F F F
166482024-02-06T22:22:50.700Z 5 100-119 152 152 152 284 284 284 F F F
166492024-02-06T22:22:50.700Z 6 120-139 152 152 152 284 284 284 F F F
166502024-02-06T22:22:50.700Z 7 140-159 152 152 152 284 284 284 F F F
166512024-02-06T22:22:50.700Z 8 160-179 153 153 153 286 286 286 F F F
166522024-02-06T22:22:50.700Z 9 180-199 151 151 151 283 283 283 F F F
166532024-02-06T22:22:50.703Z 10 200-219 144 144 144 271 271 271 F F F
166542024-02-06T22:22:50.703Z 11 220-239 153 153 153 286 286 286 F F F
166552024-02-06T22:22:50.703Z 12 240-259 147 147 147 274 274 274 F F F
166562024-02-06T22:22:50.703Z 13 260-279 150 150 150 280 280 280 F F F
166572024-02-06T22:22:50.703Z 14 280-299 150 150 150 280 280 280 F F F
166582024-02-06T22:22:50.703Z 15 300-319 150 150 150 280 280 280 F F F
166592024-02-06T22:22:50.703Z 16 320-339 153 153 153 286 286 286 F F F
166602024-02-06T22:22:50.703Z 17 340-359 151 151 151 283 283 283 F F F
166612024-02-06T22:22:50.703Z 18 360-379 145 145 145 272 272 272 F F F
166622024-02-06T22:22:50.703Z 19 380-399 152 152 152 284 284 284 F F F
166632024-02-06T22:22:50.703Z 20 400-419 153 153 153 286 286 286 F F F
166642024-02-06T22:22:50.703Z 21 420-439 151 151 151 283 283 283 F F F
166652024-02-06T22:22:50.703Z 22 440-459 150 150 150 282 282 282 F F F
166662024-02-06T22:22:50.703Z 23 460-479 149 149 149 278 278 278 F F F
166672024-02-06T22:22:50.704Z 24 480-499 145 145 145 272 272 272 F F F
166682024-02-06T22:22:50.704Z 25 500-519 149 149 149 278 278 278 F F F
166692024-02-06T22:22:50.704Z 26 520-539 149 149 149 278 278 278 F F F
166702024-02-06T22:22:50.704Z 27 540-559 153 153 153 286 286 286 F F F
166712024-02-06T22:22:50.704Z 28 560-579 150 150 150 280 280 280 F F F
166722024-02-06T22:22:50.704Z 29 580-599 151 151 151 283 283 283 F F F
166732024-02-06T22:22:50.704ZMax gen: 153, Max flush: 286
166742024-02-06T22:22:50.704Z
166752024-02-06T22:22:50.704ZBegin loop 77
166762024-02-06T22:22:50.704ZDownstairs to restart: 1
166772024-02-06T22:22:50.722Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.90331915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754}
166782024-02-06T22:22:50.751Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903566777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754}
166792024-02-06T22:22:50.751Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903609441Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754}
166802024-02-06T22:22:50.751Z{"msg":"Crucible d9de6b02-1d15-40ec-8aab-c0ac9f2d7cae has session id: 8bf73b06-13e5-4d23-80b6-4e80a0fe7439","v":0,"name":"crucible","level":30Crucible runtime is spawned
166812024-02-06T22:22:50.751Z,"time":"2024-02-06T22:22:49.903646726Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166822024-02-06T22:22:50.751Z{"msg":"Upstairs opts: Upstairs UUID: d9de6b02-1d15-40ec-8aab-c0ac9f2d7cae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903686751Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166832024-02-06T22:22:50.752Z{"msg":"Crucible stats registered with UUID: d9de6b02-1d15-40ec-8aab-c0ac9f2d7cae","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903717406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166842024-02-06T22:22:50.752Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903731385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754}
166852024-02-06T22:22:50.752Z{"msg":"d9de6b02-1d15-40ec-8aab-c0ac9f2d7cae active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903823872Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166862024-02-06T22:22:50.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903857268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166872024-02-06T22:22:50.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903887113Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166882024-02-06T22:22:50.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.90391376Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166892024-02-06T22:22:50.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903955594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166902024-02-06T22:22:50.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.903994489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166912024-02-06T22:22:50.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904021995Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166922024-02-06T22:22:50.752Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904048472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166932024-02-06T22:22:50.752Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904091816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166942024-02-06T22:22:50.752Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904130531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166952024-02-06T22:22:50.752Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904160487Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166962024-02-06T22:22:50.752Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.90420501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166972024-02-06T22:22:50.752Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904233297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166982024-02-06T22:22:50.752Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904767374Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
166992024-02-06T22:22:50.752Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904811459Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167002024-02-06T22:22:50.752Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904836595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167012024-02-06T22:22:50.752Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.904860952Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167022024-02-06T22:22:50.753Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.905433875Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167032024-02-06T22:22:50.753Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.905489857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167042024-02-06T22:22:50.753Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906073058Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167052024-02-06T22:22:50.753Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906125201Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167062024-02-06T22:22:50.753Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906147598Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167072024-02-06T22:22:50.753Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906161346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167082024-02-06T22:22:50.753Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906183303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167092024-02-06T22:22:50.753Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906197551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167102024-02-06T22:22:50.753Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.90621051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167112024-02-06T22:22:50.753Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906223948Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167122024-02-06T22:22:50.753Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.906695624Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167132024-02-06T22:22:50.753Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907119867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167142024-02-06T22:22:50.753Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907146323Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167152024-02-06T22:22:50.753Z{"msg":"[0]R flush_numbers[0..12]: [284, 284, 280, 285, 273, 284, 284, 284, 286, 283, 271, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907175629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167162024-02-06T22:22:50.753Z{"msg":"[0]R generation[0..12]: [152, 152, 150, 152, 146, 152, 152, 152, 153, 151, 144, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907206275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167172024-02-06T22:22:50.753Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907232792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167182024-02-06T22:22:50.754Z{"msg":"[1]R flush_numbers[0..12]: [284, 284, 280, 285, 273, 284, 284, 284, 286, 283, 271, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907263017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167192024-02-06T22:22:50.754Z{"msg":"[1]R generation[0..12]: [152, 152, 150, 152, 146, 152, 152, 152, 153, 151, 144, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907288484Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167202024-02-06T22:22:50.754Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
167212024-02-06T22:22:50.754Z30,"time":"2024-02-06T22:22:49.9073209Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167222024-02-06T22:22:50.754Z{"msg":"[2]R flush_numbers[0..12]: [284, 284, 280, 285, 273, 284, 284, 284, 286, 283, 271, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907347956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167232024-02-06T22:22:50.754Z{"msg":"[2]R generation[0..12]: [152, 152, 150, 152, 146, 152, 152, 152, 153, 151, 144, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907378232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167242024-02-06T22:22:50.754Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907404758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167252024-02-06T22:22:50.754Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907433724Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167262024-02-06T22:22:50.754Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.90746692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167272024-02-06T22:22:50.754Z{"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907498776Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167282024-02-06T22:22:50.754Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907519133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167292024-02-06T22:22:50.754Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.90754028Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167302024-02-06T22:22:50.754Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907568986Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167312024-02-06T22:22:50.754Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907592973Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167322024-02-06T22:22:50.754Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907620329Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167332024-02-06T22:22:50.754Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907643916Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167342024-02-06T22:22:50.754Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907669882Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167352024-02-06T22:22:50.754Z{"msg":"d9de6b02-1d15-40ec-8aab-c0ac9f2d7cae is now active with session: 8bf73b06-13e5-4d23-80b6-4e80a0fe7439","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907692969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167362024-02-06T22:22:50.754Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167372024-02-06T22:22:50.755Z,"time":"2024-02-06T22:22:49.907718436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754,"session_id":"8bf73b06-13e5-4d23-80b6-4e80a0fe7439"}
167382024-02-06T22:22:50.755Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:49.907740883Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754}
167392024-02-06T22:22:50.773ZLoading write count information from file "/tmp/test_repair_verify.data"
167402024-02-06T22:22:50.773ZRun Repair workload
167412024-02-06T22:22:50.774Z01/30 Write block 594 len 16384 data: 13 11 9 8
167422024-02-06T22:22:50.774Z02/30 Read block 308 len 4096
167432024-02-06T22:22:50.774Z03/30 Read block 439 len 36864
167442024-02-06T22:22:50.774Z04/30 Read block 437 len 24576
167452024-02-06T22:22:50.775Z05/30 Read block 580 len 4096
167462024-02-06T22:22:50.775Z06/30 Read block 9 len 36864
167472024-02-06T22:22:50.775Z07/30 Read block 119 len 12288
167482024-02-06T22:22:50.775Z08/30 Read block 592 len 20480
167492024-02-06T22:22:50.775Z09/30 Read block 585 len 12288
167502024-02-06T22:22:50.776Z10/30 Read block 142 len 28672
167512024-02-06T22:22:50.776Z11/30 Read block 248 len 16384
167522024-02-06T22:22:50.776Z12/30 Read block 205 len 8192
167532024-02-06T22:22:50.776Z13/30 Write block 460 len 12288 data: 12 11 11
167542024-02-06T22:22:50.776Z14/30 Read block 192 len 12288
167552024-02-06T22:22:50.777Z15/30 Write block 27 len 40960 data: 10 13 11 11 8 6 7 7 6 7
167562024-02-06T22:22:50.777Z16/30 Read block 283 len 20480
167572024-02-06T22:22:50.777Z17/30 Read block 378 len 40960
167582024-02-06T22:22:50.777Z18/30 Read block 401 len 12288
167592024-02-06T22:22:50.777Z19/30 Write block 406 len 20480 data: 14 13 12 14 14
167602024-02-06T22:22:50.778Z20/30 Read block 281 len 24576
167612024-02-06T22:22:50.778Z21/30 Flush
167622024-02-06T22:22:50.778Z22/30 Write block 59 len 4096 data: 14
167632024-02-06T22:22:50.778Z23/30 Read block 128 len 24576
167642024-02-06T22:22:50.778Z24/30 Write block 485 len 40960 data: 9 10 10 9 9 7 8 11 11 11
167652024-02-06T22:22:50.778Z25/30 Write block 539 len 32768 data: 7 8 10 10 11 13 11 13
167662024-02-06T22:22:50.778Z26/30 Write block 329 len 8192 data: 7 6
167672024-02-06T22:22:50.778Z27/30 Write block 115 len 8192 data: 16 15
167682024-02-06T22:22:50.778Z28/30 Write block 142 len 40960 data: 10 12 13 14 16 17 19 22 18 14
167692024-02-06T22:22:50.778Z29/30 Read block 28 len 24576
167702024-02-06T22:22:50.778Z30/30 Read block 551 len 28672
167712024-02-06T22:22:50.778Z----------------------------------------------------------------
167722024-02-06T22:22:50.779Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
167732024-02-06T22:22:50.779ZStates: Active Active Active
167742024-02-06T22:22:50.779ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167752024-02-06T22:22:50.779Z 1 Acked 1000 Write 4 Done Done Done false
167762024-02-06T22:22:50.779Z 2 Acked 1001 Read 1 Done Done Done false
167772024-02-06T22:22:50.779Z 3 Acked 1002 Read 9 Done Sent Done false
167782024-02-06T22:22:50.780Z 4 Acked 1003 Read 6 Done Sent Done false
167792024-02-06T22:22:50.780Z 5 Acked 1004 Read 1 Done Sent Done false
167802024-02-06T22:22:50.780Z 6 Acked 1005 Read 9 Done Sent Done false
167812024-02-06T22:22:50.780Z 7 Acked 1006 Read 3 Done Sent Done false
167822024-02-06T22:22:50.781Z 8 Acked 1007 Read 5 Done Sent Done false
167832024-02-06T22:22:50.781Z 9 Acked 1008 Read 3 Done Sent Done false
167842024-02-06T22:22:50.781Z 10 Acked 1009 Read 7 Done Sent Done false
167852024-02-06T22:22:50.781Z 11 Acked 1010 Read 4 Done Sent Done false
167862024-02-06T22:22:50.781Z 12 Acked 1011 Read 2 Done Sent Done false
167872024-02-06T22:22:50.781Z 13 Acked 1012 Write 3 Done Sent Done false
167882024-02-06T22:22:50.781Z 14 Acked 1013 Read 3 Done Sent Done false
167892024-02-06T22:22:50.782Z 15 Acked 1014 Write 10 Done Sent Done false
167902024-02-06T22:22:50.782Z 16 Acked 1015 Read 5 Done Sent Done false
167912024-02-06T22:22:50.782Z 17 Acked 1016 Read 10 Done Sent Done false
167922024-02-06T22:22:50.782Z 18 Acked 1017 Read 3 Done Sent Done false
167932024-02-06T22:22:50.782Z 19 Acked 1018 Write 5 Done Sent Done false
167942024-02-06T22:22:50.783Z 20 Acked 1019 Read 6 Done Sent Done false
167952024-02-06T22:22:50.783Z 21 Acked 1020 Flush 0 Done Sent Done false
167962024-02-06T22:22:50.783Z 22 Acked 1021 Write 1 Done Sent Done false
167972024-02-06T22:22:50.783Z 23 Acked 1022 Read 6 Done Sent Done false
167982024-02-06T22:22:50.783Z 24 Acked 1023 Write 10 Done Sent Done false
167992024-02-06T22:22:50.783Z 25 Acked 1024 Write 8 Done Sent Done false
168002024-02-06T22:22:50.783Z 26 Acked 1025 Write 2 Done Sent Done false
168012024-02-06T22:22:50.784Z 27 Acked 1026 Write 2 Done Sent Done false
168022024-02-06T22:22:50.784Z 28 Acked 1027 Write 10 Done Sent Done false
168032024-02-06T22:22:50.784Z 29 Acked 1028 Read 6 Done Sent Done false
168042024-02-06T22:22:50.784Z 30 Acked 1029 Read 7 Done Sent Done false
168052024-02-06T22:22:50.784Z STATES DS:0 DS:1 DS:2 TOTAL
168062024-02-06T22:22:50.785Z New 0 0 0 0
168072024-02-06T22:22:50.785Z Sent 0 28 0 28
168082024-02-06T22:22:50.785Z Done 30 2 30 62
168092024-02-06T22:22:50.785Z Skipped 0 0 0 0
168102024-02-06T22:22:50.785Z Error 0 0 0 0
168112024-02-06T22:22:50.785ZLast Flush: 1020 0 1020
168122024-02-06T22:22:50.786ZDownstairs last five completed:
168132024-02-06T22:22:50.786ZUpstairs last five completed: 30 29 28 27 26
168142024-02-06T22:22:50.786Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:49.935771816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1754}
168152024-02-06T22:22:50.786ZWrote out file "/tmp/test_repair_verify.data"
168162024-02-06T22:22:50.786Z
168172024-02-06T22:22:50.787ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168182024-02-06T22:22:50.793ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168192024-02-06T22:22:50.793Z 0 000-019 152 152 152 284 284 284 F F F
168202024-02-06T22:22:50.793Z 1 020-039 154 152 154 287 284 287 F F F <---
168212024-02-06T22:22:50.793Z 2 040-059 150 150 150 280 280 280 T F T <---
168222024-02-06T22:22:50.794Z 3 060-079 152 152 152 285 285 285 F F F
168232024-02-06T22:22:50.794Z 4 080-099 146 146 146 273 273 273 F F F
168242024-02-06T22:22:50.794Z 5 100-119 152 152 152 284 284 284 T F T <---
168252024-02-06T22:22:50.794Z 6 120-139 152 152 152 284 284 284 F F F
168262024-02-06T22:22:50.794Z 7 140-159 152 152 152 284 284 284 T F T <---
168272024-02-06T22:22:50.794Z 8 160-179 153 153 153 286 286 286 F F F
168282024-02-06T22:22:50.794Z 9 180-199 151 151 151 283 283 283 F F F
168292024-02-06T22:22:50.795Z 10 200-219 144 144 144 271 271 271 F F F
168302024-02-06T22:22:50.795Z 11 220-239 153 153 153 286 286 286 F F F
168312024-02-06T22:22:50.795Z 12 240-259 147 147 147 274 274 274 F F F
168322024-02-06T22:22:50.795Z 13 260-279 150 150 150 280 280 280 F F F
168332024-02-06T22:22:50.795Z 14 280-299 150 150 150 280 280 280 F F F
168342024-02-06T22:22:50.795Z 15 300-319 150 150 150 280 280 280 F F F
168352024-02-06T22:22:50.795Z 16 320-339 153 153 153 286 286 286 T F T <---
168362024-02-06T22:22:50.795Z 17 340-359 151 151 151 283 283 283 F F F
168372024-02-06T22:22:50.795Z 18 360-379 145 145 145 272 272 272 F F F
168382024-02-06T22:22:50.795Z 19 380-399 152 152 152 284 284 284 F F F
168392024-02-06T22:22:50.795Z 20 400-419 154 153 154 287 286 287 F F F <---
168402024-02-06T22:22:50.795Z 21 420-439 151 151 151 283 283 283 F F F
168412024-02-06T22:22:50.795Z 22 440-459 150 150 150 282 282 282 F F F
168422024-02-06T22:22:50.795Z 23 460-479 154 149 154 287 278 287 F F F <---
168432024-02-06T22:22:50.795Z 24 480-499 145 145 145 272 272 272 T F T <---
168442024-02-06T22:22:50.795Z 25 500-519 149 149 149 278 278 278 F F F
168452024-02-06T22:22:50.795Z 26 520-539 149 149 149 278 278 278 T F T <---
168462024-02-06T22:22:50.795Z 27 540-559 153 153 153 286 286 286 T F T <---
168472024-02-06T22:22:50.795Z 28 560-579 150 150 150 280 280 280 F F F
168482024-02-06T22:22:50.796Z 29 580-599 154 151 154 287 283 287 F T F <---
168492024-02-06T22:22:50.796ZMax gen: 154, Max flush: 287
168502024-02-06T22:22:50.796ZError: Difference in extent metadata found!
168512024-02-06T22:22:50.796ZA Difference in extent metadata is expected here
168522024-02-06T22:22:50.796ZOn loop 77
168532024-02-06T22:22:50.796Z
168542024-02-06T22:22:50.797ZVerifying data now
168552024-02-06T22:22:55.346ZNo change after verify
168562024-02-06T22:22:55.346ZLoop: 77 Downstairs dump after verify (and repair):
168572024-02-06T22:22:55.359ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168582024-02-06T22:22:55.359Z 0 000-019 152 152 152 284 284 284 F F F
168592024-02-06T22:22:55.359Z 1 020-039 154 154 154 287 287 287 F F F
168602024-02-06T22:22:55.359Z 2 040-059 155 155 155 288 288 288 F F F
168612024-02-06T22:22:55.359Z 3 060-079 152 152 152 285 285 285 F F F
168622024-02-06T22:22:55.360Z 4 080-099 146 146 146 273 273 273 F F F
168632024-02-06T22:22:55.360Z 5 100-119 155 155 155 288 288 288 F F F
168642024-02-06T22:22:55.360Z 6 120-139 152 152 152 284 284 284 F F F
168652024-02-06T22:22:55.360Z 7 140-159 155 155 155 288 288 288 F F F
168662024-02-06T22:22:55.360Z 8 160-179 153 153 153 286 286 286 F F F
168672024-02-06T22:22:55.364Z 9 180-199 151 151 151 283 283 283 F F F
168682024-02-06T22:22:55.364Z 10 200-219 144 144 144 271 271 271 F F F
168692024-02-06T22:22:55.364Z 11 220-239 153 153 153 286 286 286 F F F
168702024-02-06T22:22:55.364Z 12 240-259 147 147 147 274 274 274 F F F
168712024-02-06T22:22:55.364Z 13 260-279 150 150 150 280 280 280 F F F
168722024-02-06T22:22:55.364Z 14 280-299 150 150 150 280 280 280 F F F
168732024-02-06T22:22:55.364Z 15 300-319 150 150 150 280 280 280 F F F
168742024-02-06T22:22:55.364Z 16 320-339 155 155 155 288 288 288 F F F
168752024-02-06T22:22:55.365Z 17 340-359 151 151 151 283 283 283 F F F
168762024-02-06T22:22:55.365Z 18 360-379 145 145 145 272 272 272 F F F
168772024-02-06T22:22:55.365Z 19 380-399 152 152 152 284 284 284 F F F
168782024-02-06T22:22:55.365Z 20 400-419 154 154 154 287 287 287 F F F
168792024-02-06T22:22:55.365Z 21 420-439 151 151 151 283 283 283 F F F
168802024-02-06T22:22:55.365Z 22 440-459 150 150 150 282 282 282 F F F
168812024-02-06T22:22:55.365Z 23 460-479 154 154 154 287 287 287 F F F
168822024-02-06T22:22:55.365Z 24 480-499 155 155 155 288 288 288 F F F
168832024-02-06T22:22:55.365Z 25 500-519 149 149 149 278 278 278 F F F
168842024-02-06T22:22:55.365Z 26 520-539 155 155 155 288 288 288 F F F
168852024-02-06T22:22:55.365Z 27 540-559 155 155 155 288 288 288 F F F
168862024-02-06T22:22:55.365Z 28 560-579 150 150 150 280 280 280 F F F
168872024-02-06T22:22:55.366Z 29 580-599 154 154 154 287 287 287 F F F
168882024-02-06T22:22:55.366ZMax gen: 155, Max flush: 288
168892024-02-06T22:22:55.366Z
168902024-02-06T22:22:55.366ZBegin loop 78
168912024-02-06T22:22:55.366ZDownstairs to restart: 0
168922024-02-06T22:22:55.382Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.560609583Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762}
168932024-02-06T22:22:55.382Z{Crucible runtime is spawned
168942024-02-06T22:22:55.388Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.560910443Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762}
168952024-02-06T22:22:55.388Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.560950427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762}
168962024-02-06T22:22:55.388Z{"msg":"Crucible f879a613-21db-432a-856d-bdcf9e56d47c has session id: 487bea03-81e6-4df3-a610-356e0aef5a0b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.560977054Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
168972024-02-06T22:22:55.388Z{"msg":"Upstairs opts: Upstairs UUID: f879a613-21db-432a-856d-bdcf9e56d47c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561030956Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
168982024-02-06T22:22:55.388Z{"msg":"Crucible stats registered with UUID: f879a613-21db-432a-856d-bdcf9e56d47c","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561067171Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
168992024-02-06T22:22:55.388Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561082489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762}
169002024-02-06T22:22:55.388Z{"msg":"f879a613-21db-432a-856d-bdcf9e56d47c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561101637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169012024-02-06T22:22:55.388Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561114795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169022024-02-06T22:22:55.388Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561129563Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169032024-02-06T22:22:55.388Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.56115062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169042024-02-06T22:22:55.388Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561170037Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169052024-02-06T22:22:55.388Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561184426Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169062024-02-06T22:22:55.388Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.56122419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169072024-02-06T22:22:55.388Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561262695Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169082024-02-06T22:22:55.388Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.56130261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169092024-02-06T22:22:55.389Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561323077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169102024-02-06T22:22:55.389Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561456559Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169112024-02-06T22:22:55.389Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.56152084Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169122024-02-06T22:22:55.389Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:54.561548077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169132024-02-06T22:22:55.389Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:54.561571053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169142024-02-06T22:22:55.389Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:54.561737531Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169152024-02-06T22:22:55.389Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561791504Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169162024-02-06T22:22:55.389Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.561841857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169172024-02-06T22:22:55.389Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.56188983Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169182024-02-06T22:22:55.389Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.562302784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169192024-02-06T22:22:55.389Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.56233758Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169202024-02-06T22:22:55.389Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.562354088Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169212024-02-06T22:22:55.389Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.562386853Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169222024-02-06T22:22:55.389Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.562904043Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169232024-02-06T22:22:55.389Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.562962405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169242024-02-06T22:22:55.389Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.563608548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169252024-02-06T22:22:55.389Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.563648363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169262024-02-06T22:22:55.389Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.563665371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169272024-02-06T22:22:55.389Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.563680098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169282024-02-06T22:22:55.389Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.563702146Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169292024-02-06T22:22:55.389Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:54.563716924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169302024-02-06T22:22:56.385Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.562371163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169312024-02-06T22:22:56.390Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.562599592Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169322024-02-06T22:22:56.390Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.563468195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169332024-02-06T22:22:56.390Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.563514149Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169342024-02-06T22:22:56.390Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.563983905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169352024-02-06T22:22:56.390Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564526952Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169362024-02-06T22:22:56.390Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564574056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169372024-02-06T22:22:56.390Z{"msg":"[0]R flush_numbers[0..12]: [284, 287, 288, 285, 273, 288, 284, 288, 286, 283, 271, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564600982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169382024-02-06T22:22:56.390Z{"msg":"[0]R generation[0..12]: [152, 154, 155, 152, 146, 155, 152, 155, 153, 151, 144, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564636837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169392024-02-06T22:22:56.390Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564674052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169402024-02-06T22:22:56.390Z{"msg":"[1]R flush_numbers[0..12]: [284, 287, 288, 285, 273, 288, 284, 288, 286, 283, 271, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564710907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169412024-02-06T22:22:56.390Z{"msg":"[1]R generation[0..12]: [152, 154, 155, 152, 146, 155, 152, 155, 153, 151, 144, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564747392Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
169422024-02-06T22:22:56.391Z,"":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169432024-02-06T22:22:56.391Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564790376Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169442024-02-06T22:22:56.391Z{"msg":"[2]R flush_numbers[0..12]: [284, 287, 288, 285, 273, 288, 284, 288, 286, 283, 271, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564830851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169452024-02-06T22:22:56.391Z{"msg":"[2]R generation[0..12]: [152, 154, 155, 152, 146, 155, 152, 155, 153, 151, 144, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564870096Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169462024-02-06T22:22:56.391Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.56490957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169472024-02-06T22:22:56.391Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.564947735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169482024-02-06T22:22:56.391Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.5649859Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169492024-02-06T22:22:56.391Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565050151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169502024-02-06T22:22:56.391Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565088256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169512024-02-06T22:22:56.391Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565124941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169522024-02-06T22:22:56.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565159866Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169532024-02-06T22:22:56.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565196262Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169542024-02-06T22:22:56.391ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169552024-02-06T22:22:56.391Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565235116Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169562024-02-06T22:22:56.391Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565268552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169572024-02-06T22:22:56.391Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.56528309Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169582024-02-06T22:22:56.391Z{"msg":"f879a613-21db-432a-856d-bdcf9e56d47c is now active with session: 487bea03-81e6-4df3-a610-356e0aef5a0b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565304097Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169592024-02-06T22:22:56.391Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565316815Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762,"session_id":"487bea03-81e6-4df3-a610-356e0aef5a0b"}
169602024-02-06T22:22:56.391Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:22:55.565336953Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762}
169612024-02-06T22:22:56.391ZLoading write count information from file "/tmp/test_repair_verify.data"
169622024-02-06T22:22:56.391ZRun Repair workload
169632024-02-06T22:22:56.391Z01/30 Write block 180 len 32768 data: 16 17 15 19 20 20 19 18
169642024-02-06T22:22:56.395Z02/30 Write block 162 len 4096 data: 16
169652024-02-06T22:22:56.395Z03/30 Read block 503 len 16384
169662024-02-06T22:22:56.395Z04/30 Read block 484 len 8192
169672024-02-06T22:22:56.395Z05/30 Read block 186 len 32768
169682024-02-06T22:22:56.395Z06/30 Read block 248 len 24576
169692024-02-06T22:22:56.395Z07/30 Read block 428 len 24576
169702024-02-06T22:22:56.395Z08/30 Write block 386 len 36864 data: 13 9 9 8 8 8 8 8 7
169712024-02-06T22:22:56.395Z09/30 Read block 328 len 28672
169722024-02-06T22:22:56.398Z10/30 Write block 561 len 4096 data: 9
169732024-02-06T22:22:56.398Z11/30 Read block 81 len 20480
169742024-02-06T22:22:56.398Z12/30 Read block 594 len 24576
169752024-02-06T22:22:56.398Z13/30 Write block 384 len 4096 data: 10
169762024-02-06T22:22:56.401Z14/30 Read block 25 len 16384
169772024-02-06T22:22:56.401Z15/30 Read block 444 len 12288
169782024-02-06T22:22:56.401Z16/30 Write block 209 len 8192 data: 12 12
169792024-02-06T22:22:56.401Z17/30 Read block 453 len 8192
169802024-02-06T22:22:56.401Z18/30 Read block 339 len 20480
169812024-02-06T22:22:56.404Z19/30 Read block 137 len 28672
169822024-02-06T22:22:56.404Z20/30 Write block 310 len 8192 data: 13 13
169832024-02-06T22:22:56.404Z21/30 Read block 380 len 36864
169842024-02-06T22:22:56.407Z22/30 Write block 54 len 28672 data: 24 22 21 17 15 15 15
169852024-02-06T22:22:56.407Z23/30 Read block 547 len 8192
169862024-02-06T22:22:56.407Z24/30 Flush
169872024-02-06T22:22:56.411Z25/30 Write block 84 len 32768 data: 9 9 10 12 11 11 9 9
169882024-02-06T22:22:56.411Z26/30 Flush
169892024-02-06T22:22:56.413Z27/30 Write block 439 len 4096 data: 13
169902024-02-06T22:22:56.414Z28/30 Read block 534 len 32768
169912024-02-06T22:22:56.417Z29/30 Read block 379 len 24576
169922024-02-06T22:22:56.417Z30/30 Write block 153 len 8192 data: 17 16
169932024-02-06T22:22:56.417Z----------------------------------------------------------------
169942024-02-06T22:22:56.417Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
169952024-02-06T22:22:56.417ZStates: Active Active Active
169962024-02-06T22:22:56.418ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169972024-02-06T22:22:56.418Z 1 Acked 1000 Write 8 Sent Done Done false
169982024-02-06T22:22:56.418Z 2 Acked 1001 Write 1 Sent Done Done false
169992024-02-06T22:22:56.418Z 3 Acked 1002 Read 4 Sent Done Done false
170002024-02-06T22:22:56.418Z 4 Acked 1003 Read 2 Sent Done Done false
170012024-02-06T22:22:56.418Z 5 Acked 1004 Read 8 Sent Done Done false
170022024-02-06T22:22:56.418Z 6 Acked 1005 Read 6 Sent Done Done false
170032024-02-06T22:22:56.418Z 7 Acked 1006 Read 6 Sent Done Done false
170042024-02-06T22:22:56.418Z 8 Acked 1007 Write 9 Sent Done Done false
170052024-02-06T22:22:56.418Z 9 Acked 1008 Read 7 Sent Done Done false
170062024-02-06T22:22:56.419Z 10 Acked 1009 Write 1 Sent Done Done false
170072024-02-06T22:22:56.419Z 11 Acked 1010 Read 5 Sent Done Done false
170082024-02-06T22:22:56.419Z 12 Acked 1011 Read 6 Sent Done Done false
170092024-02-06T22:22:56.419Z 13 Acked 1012 Write 1 Sent Done Done false
170102024-02-06T22:22:56.419Z 14 Acked 1013 Read 4 Sent Done Done false
170112024-02-06T22:22:56.419Z 15 Acked 1014 Read 3 Sent Done Done false
170122024-02-06T22:22:56.419Z 16 Acked 1015 Write 2 Sent Done Done false
170132024-02-06T22:22:56.419Z 17 Acked 1016 Read 2 Sent Done Done false
170142024-02-06T22:22:56.419Z 18 Acked 1017 Read 5 Sent Done Done false
170152024-02-06T22:22:56.419Z 19 Acked 1018 Read 7 Sent Done Done false
170162024-02-06T22:22:56.419Z 20 Acked 1019 Write 2 Sent Done Done false
170172024-02-06T22:22:56.420Z 21 Acked 1020 Read 9 Sent Done Done false
170182024-02-06T22:22:56.420Z 22 Acked 1021 Write 7 Sent Done Done false
170192024-02-06T22:22:56.420Z 23 Acked 1022 Read 2 Sent Done Done false
170202024-02-06T22:22:56.420Z 24 Acked 1023 Flush 0 Sent Done Done false
170212024-02-06T22:22:56.420Z 25 Acked 1024 Write 8 Sent Done Done false
170222024-02-06T22:22:56.420Z 26 Acked 1025 Flush 0 Sent Done Done false
170232024-02-06T22:22:56.420Z 27 Acked 1026 Write 1 Sent Done Done false
170242024-02-06T22:22:56.420Z 28 Acked 1027 Read 8 Sent Done Done false
170252024-02-06T22:22:56.420Z 29 Acked 1028 Read 6 Sent Done Done false
170262024-02-06T22:22:56.421Z 30 Acked 1029 Write 2 Sent Sent Sent false
170272024-02-06T22:22:56.421Z STATES DS:0 DS:1 DS:2 TOTAL
170282024-02-06T22:22:56.421Z New 0 0 0 0
170292024-02-06T22:22:56.421Z Sent 30 1 1 32
170302024-02-06T22:22:56.421Z Done 0 29 29 58
170312024-02-06T22:22:56.421Z Skipped 0 0 0 0
170322024-02-06T22:22:56.421Z Error 0 0 0 0
170332024-02-06T22:22:56.422ZLast Flush: 0 1025 1025
170342024-02-06T22:22:56.422ZDownstairs last five completed:
170352024-02-06T22:22:56.422ZUpstairs last five completed: 30 29 28 27 26
170362024-02-06T22:22:56.422Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:22:55.597055837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1762}
170372024-02-06T22:22:56.422ZWrote out file "/tmp/test_repair_verify.data"
170382024-02-06T22:22:56.435Z
170392024-02-06T22:22:56.435ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170402024-02-06T22:22:56.440ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170412024-02-06T22:22:56.440Z 0 000-019 152 152 152 284 284 284 F F F
170422024-02-06T22:22:56.440Z 1 020-039 154 154 154 287 287 287 F F F
170432024-02-06T22:22:56.440Z 2 040-059 155 156 156 288 289 289 F F F <---
170442024-02-06T22:22:56.440Z 3 060-079 152 156 156 285 289 289 F F F <---
170452024-02-06T22:22:56.440Z 4 080-099 146 156 156 273 290 290 F F F <---
170462024-02-06T22:22:56.440Z 5 100-119 155 155 155 288 288 288 F F F
170472024-02-06T22:22:56.570Z 6 120-139 152 152 152 284 284 284 F F F
170482024-02-06T22:22:56.570Z 7 140-159 155 155 155 288 288 288 F T T <---
170492024-02-06T22:22:56.570Z 8 160-179 153 156 156 286 289 289 F F F <---
170502024-02-06T22:22:56.570Z 9 180-199 151 156 156 283 289 289 F F F <---
170512024-02-06T22:22:56.570Z 10 200-219 144 156 156 271 289 289 F F F <---
170522024-02-06T22:22:56.570Z 11 220-239 153 153 153 286 286 286 F F F
170532024-02-06T22:22:56.570Z 12 240-259 147 147 147 274 274 274 F F F
170542024-02-06T22:22:56.570Z 13 260-279 150 150 150 280 280 280 F F F
170552024-02-06T22:22:56.570Z 14 280-299 150 150 150 280 280 280 F F F
170562024-02-06T22:22:56.570Z 15 300-319 150 156 156 280 289 289 F F F <---
170572024-02-06T22:22:56.571Z 16 320-339 155 155 155 288 288 288 F F F
170582024-02-06T22:22:56.571Z 17 340-359 151 151 151 283 283 283 F F F
170592024-02-06T22:22:56.571Z 18 360-379 145 145 145 272 272 272 F F F
170602024-02-06T22:22:56.571Z 19 380-399 152 156 156 284 289 289 F F F <---
170612024-02-06T22:22:56.571Z 20 400-419 154 154 154 287 287 287 F F F
170622024-02-06T22:22:56.571Z 21 420-439 151 151 151 283 283 283 F T T <---
170632024-02-06T22:22:56.571Z 22 440-459 150 150 150 282 282 282 F F F
170642024-02-06T22:22:56.571Z 23 460-479 154 154 154 287 287 287 F F F
170652024-02-06T22:22:56.571Z 24 480-499 155 155 155 288 288 288 F F F
170662024-02-06T22:22:56.571Z 25 500-519 149 149 149 278 278 278 F F F
170672024-02-06T22:22:56.571Z 26 520-539 155 155 155 288 288 288 F F F
170682024-02-06T22:22:56.571Z 27 540-559 155 155 155 288 288 288 F F F
170692024-02-06T22:22:56.571Z 28 560-579 150 156 156 280 289 289 F F F <---
170702024-02-06T22:22:56.571Z 29 580-599 154 154 154 287 287 287 F F F
170712024-02-06T22:22:56.571ZMax gen: 156, Max flush: 290
170722024-02-06T22:22:56.571ZError: Difference in extent metadata found!
170732024-02-06T22:22:56.571ZA Difference in extent metadata is expected here
170742024-02-06T22:22:56.571ZOn loop 78
170752024-02-06T22:22:56.571Z
170762024-02-06T22:22:56.577ZVerifying data now
170772024-02-06T22:23:00.996ZNo change after verify
170782024-02-06T22:23:00.996ZLoop: 78 Downstairs dump after verify (and repair):
170792024-02-06T22:23:01.011ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170802024-02-06T22:23:01.011Z 0 000-019 152 152 152 284 284 284 F F F
170812024-02-06T22:23:01.011Z 1 020-039 154 154 154 287 287 287 F F F
170822024-02-06T22:23:01.011Z 2 040-059 156 156 156 289 289 289 F F F
170832024-02-06T22:23:01.011Z 3 060-079 156 156 156 289 289 289 F F F
170842024-02-06T22:23:01.011Z 4 080-099 156 156 156 290 290 290 F F F
170852024-02-06T22:23:01.011Z 5 100-119 155 155 155 288 288 288 F F F
170862024-02-06T22:23:01.011Z 6 120-139 152 152 152 284 284 284 F F F
170872024-02-06T22:23:01.015Z 7 140-159 157 157 157 291 291 291 F F F
170882024-02-06T22:23:01.015Z 8 160-179 156 156 156 289 289 289 F F F
170892024-02-06T22:23:01.015Z 9 180-199 156 156 156 289 289 289 F F F
170902024-02-06T22:23:01.015Z 10 200-219 156 156 156 289 289 289 F F F
170912024-02-06T22:23:01.015Z 11 220-239 153 153 153 286 286 286 F F F
170922024-02-06T22:23:01.015Z 12 240-259 147 147 147 274 274 274 F F F
170932024-02-06T22:23:01.015Z 13 260-279 150 150 150 280 280 280 F F F
170942024-02-06T22:23:01.015Z 14 280-299 150 150 150 280 280 280 F F F
170952024-02-06T22:23:01.015Z 15 300-319 156 156 156 289 289 289 F F F
170962024-02-06T22:23:01.015Z 16 320-339 155 155 155 288 288 288 F F F
170972024-02-06T22:23:01.015Z 17 340-359 151 151 151 283 283 283 F F F
170982024-02-06T22:23:01.015Z 18 360-379 145 145 145 272 272 272 F F F
170992024-02-06T22:23:01.016Z 19 380-399 156 156 156 289 289 289 F F F
171002024-02-06T22:23:01.016Z 20 400-419 154 154 154 287 287 287 F F F
171012024-02-06T22:23:01.016Z 21 420-439 157 157 157 291 291 291 F F F
171022024-02-06T22:23:01.016Z 22 440-459 150 150 150 282 282 282 F F F
171032024-02-06T22:23:01.016Z 23 460-479 154 154 154 287 287 287 F F F
171042024-02-06T22:23:01.016Z 24 480-499 155 155 155 288 288 288 F F F
171052024-02-06T22:23:01.016Z 25 500-519 149 149 149 278 278 278 F F F
171062024-02-06T22:23:01.016Z 26 520-539 155 155 155 288 288 288 F F F
171072024-02-06T22:23:01.016Z 27 540-559 155 155 155 288 288 288 F F F
171082024-02-06T22:23:01.016Z 28 560-579 156 156 156 289 289 289 F F F
171092024-02-06T22:23:01.016Z 29 580-599 154 154 154 287 287 287 F F F
171102024-02-06T22:23:01.016ZMax gen: 157, Max flush: 291
171112024-02-06T22:23:01.016Z
171122024-02-06T22:23:01.016ZBegin loop 79
171132024-02-06T22:23:01.016ZDownstairs to restart: 2
171142024-02-06T22:23:01.032Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212186118Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770}
171152024-02-06T22:23:01.032Z{Crucible runtime is spawned
171162024-02-06T22:23:01.037Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.2125402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770}
171172024-02-06T22:23:01.037Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212581184Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770}
171182024-02-06T22:23:01.038Z{"msg":"Crucible 91af04ce-e8d6-4f5a-96bd-7b82e7f1e2ea has session id: b86dc8de-8f5a-437d-9e00-9bb3f774d9c7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.21261177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171192024-02-06T22:23:01.038Z{"msg":"Upstairs opts: Upstairs UUID: 91af04ce-e8d6-4f5a-96bd-7b82e7f1e2ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212671432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171202024-02-06T22:23:01.038Z{"msg":"Crucible stats registered with UUID: 91af04ce-e8d6-4f5a-96bd-7b82e7f1e2ea","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212717406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171212024-02-06T22:23:01.038Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.2127629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770}
171222024-02-06T22:23:01.038Z{"msg":"91af04ce-e8d6-4f5a-96bd-7b82e7f1e2ea active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212803205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171232024-02-06T22:23:01.038Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212857157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171242024-02-06T22:23:01.038Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.21290895Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171252024-02-06T22:23:01.038Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212943806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171262024-02-06T22:23:01.038Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.212976341Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171272024-02-06T22:23:01.038Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213008527Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171282024-02-06T22:23:01.038Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213041442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171292024-02-06T22:23:01.038Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213073898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171302024-02-06T22:23:01.038Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213114592Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171312024-02-06T22:23:01.038Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213148488Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171322024-02-06T22:23:01.038Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213182403Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171332024-02-06T22:23:01.038Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.21320661Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171342024-02-06T22:23:01.038Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213251104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171352024-02-06T22:23:01.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213813468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171362024-02-06T22:23:01.038Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213851273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171372024-02-06T22:23:01.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.21386927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171382024-02-06T22:23:01.038Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.213892207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171392024-02-06T22:23:01.038Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.214520862Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171402024-02-06T22:23:01.038Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.214562957Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171412024-02-06T22:23:01.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215119812Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171422024-02-06T22:23:01.039Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215156927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171432024-02-06T22:23:01.039Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215304007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171442024-02-06T22:23:01.039Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215345271Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171452024-02-06T22:23:01.039Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215360879Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171462024-02-06T22:23:01.039Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215373717Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171472024-02-06T22:23:01.039Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215395334Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171482024-02-06T22:23:01.039Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215416391Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171492024-02-06T22:23:01.039Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.215735788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171502024-02-06T22:23:01.043Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216208954Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171512024-02-06T22:23:01.043Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216228122Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171522024-02-06T22:23:01.043Z{"msg":"[0]R flush_numbers[0..12]: [284, 287, 289, 289, 290, 288, 284, 291, 289, 289, 289, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216286354Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171532024-02-06T22:23:01.043Z{"msg":"[0]R generation[0..12]: [152, 154, 156, 156, 156, 155, 152, 157, 156, 156, 156, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.21631214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171542024-02-06T22:23:01.043Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216327768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171552024-02-06T22:23:01.043Z{"msg":"[1]R flush_numbers[0..12]: [284, 287, 289, 289, 290, 288, 284, 291, 289, 289, 289, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216349415Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171562024-02-06T22:23:01.043Z{"msg":"[1]R generation[0..12]: [152, 154, 156, 156, 156, 155, 152, 157, 156, 156, 156, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216363713Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171572024-02-06T22:23:01.043Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.21638429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171582024-02-06T22:23:01.043Z{"msg":"[2]R flush_numbers[0..12]: [284, 287, 289, 289, 290, 288, 284, 291, 289, 289, 289, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216398729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171592024-02-06T22:23:01.043Z{"msg":"[2]R generation[0..12]: [152, 154, 156, 156, 156, 155, 152, 157, 156, 156, 156, 153]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
171602024-02-06T22:23:01.044Z,"time":"2024-02-06T22:23:00.216419706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171612024-02-06T22:23:01.044Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216436974Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171622024-02-06T22:23:01.044Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216457621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171632024-02-06T22:23:01.044Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216471849Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171642024-02-06T22:23:01.044Z{"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216491966Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171652024-02-06T22:23:01.044Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216505584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171662024-02-06T22:23:01.044Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216519812Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171672024-02-06T22:23:01.044Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216532191Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171682024-02-06T22:23:01.044Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216547619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171692024-02-06T22:23:01.044Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216574035Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171702024-02-06T22:23:01.044Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216588933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171712024-02-06T22:23:01.044Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.2166086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171722024-02-06T22:23:01.044Z{"msg":"91af04ce-e8d6-4f5a-96bd-7b82e7f1e2ea is now active with session: b86dc8de-8f5a-437d-9e00-9bb3f774d9c7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216622058Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171732024-02-06T22:23:01.044Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216646635Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770,"session_id":"b86dc8de-8f5a-437d-9e00-9bb3f774d9c7"}
171742024-02-06T22:23:01.044Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:00.216659424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770}
171752024-02-06T22:23:01.044ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171762024-02-06T22:23:01.044ZLoading write count information from file "/tmp/test_repair_verify.data"
171772024-02-06T22:23:01.044ZRun Repair workload
171782024-02-06T22:23:01.044Z01/30 Write block 122 len 8192 data: 14 16
171792024-02-06T22:23:01.044Z02/30 Write block 401 len 36864 data: 13 14 14 13 14 15 14 13 15
171802024-02-06T22:23:01.044Z03/30 Read block 137 len 40960
171812024-02-06T22:23:01.044Z04/30 Write block 159 len 40960 data: 15 13 13 17 14 14 11 11 15 18
171822024-02-06T22:23:01.045Z05/30 Read block 267 len 4096
171832024-02-06T22:23:01.048Z06/30 Write block 257 len 36864 data: 9 11 12 13 18 19 20 18 20
171842024-02-06T22:23:01.048Z07/30 Write block 115 len 20480 data: 17 16 17 16 14
171852024-02-06T22:23:01.048Z08/30 Read block 316 len 40960
171862024-02-06T22:23:01.048Z09/30 Write block 234 len 36864 data: 14 15 14 13 8 8 7 9 9
171872024-02-06T22:23:01.048Z10/30 Write block 166 len 4096 data: 12
171882024-02-06T22:23:01.048Z11/30 Read block 248 len 4096
171892024-02-06T22:23:01.051Z12/30 Read block 382 len 4096
171902024-02-06T22:23:01.051Z13/30 Write block 130 len 28672 data: 16 15 13 14 12 10 9
171912024-02-06T22:23:01.051Z14/30 Read block 483 len 12288
171922024-02-06T22:23:01.051Z15/30 Write block 505 len 28672 data: 14 15 15 14 14 13 12
171932024-02-06T22:23:01.055Z16/30 Flush
171942024-02-06T22:23:01.059Z17/30 Write block 579 len 16384 data: 15 14 11 8
171952024-02-06T22:23:01.059Z18/30 Write block 95 len 20480 data: 8 8 9 10 10
171962024-02-06T22:23:01.059Z19/30 Write block 449 len 36864 data: 8 8 9 8 9 14 13 9 9
171972024-02-06T22:23:01.059Z20/30 Read block 38 len 28672
171982024-02-06T22:23:01.069Z21/30 Write block 42 len 28672 data: 10 10 10 10 11 11 12
171992024-02-06T22:23:01.069Z22/30 Read block 323 len 40960
172002024-02-06T22:23:01.073Z23/30 Write block 581 len 20480 data: 12 9 9 12 14
172012024-02-06T22:23:01.073Z24/30 Write block 182 len 12288 data: 16 20 21
172022024-02-06T22:23:01.073Z25/30 Read block 267 len 8192
172032024-02-06T22:23:01.073Z26/30 Write block 419 len 40960 data: 16 16 14 13 11 12 12 13 11 13
172042024-02-06T22:23:01.073Z27/30 Write block 65 len 12288 data: 9 10 9
172052024-02-06T22:23:01.073Z28/30 Write block 221 len 24576 data: 16 19 19 18 15 15
172062024-02-06T22:23:01.073Z29/30 Read block 417 len 36864
172072024-02-06T22:23:01.073Z30/30 Write block 491 len 32768 data: 9 12 12 12 12 14 13 13
172082024-02-06T22:23:01.073Z----------------------------------------------------------------
172092024-02-06T22:23:01.073Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
172102024-02-06T22:23:01.073ZStates: Active Active Active
172112024-02-06T22:23:01.073ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172122024-02-06T22:23:01.073Z 1 Acked 1000 Write 2 Done Done Done false
172132024-02-06T22:23:01.073Z 2 Acked 1001 Write 9 Done Done Done false
172142024-02-06T22:23:01.073Z 3 Acked 1002 Read 10 Done Done Done false
172152024-02-06T22:23:01.073Z 4 Acked 1003 Write 10 Done Done Done false
172162024-02-06T22:23:01.073Z 5 Acked 1004 Read 1 Done Done Done false
172172024-02-06T22:23:01.073Z 6 Acked 1005 Write 9 Done Done Done false
172182024-02-06T22:23:01.073Z 7 Acked 1006 Write 5 Done Done Done false
172192024-02-06T22:23:01.073Z 8 Acked 1007 Read 10 Done Done Done false
172202024-02-06T22:23:01.073Z 9 Acked 1008 Write 9 Done Done Done false
172212024-02-06T22:23:01.073Z 10 Acked 1009 Write 1 Done Done Done false
172222024-02-06T22:23:01.074Z 11 Acked 1010 Read 1 Done Done Done false
172232024-02-06T22:23:01.074Z 12 Acked 1011 Read 1 Done Done Done false
172242024-02-06T22:23:01.074Z 13 Acked 1012 Write 7 Done Done Done false
172252024-02-06T22:23:01.074Z 14 Acked 1013 Read 3 Done Done Done false
172262024-02-06T22:23:01.074Z 15 Acked 1014 Write 7 Done Done Sent false
172272024-02-06T22:23:01.074Z 16 Acked 1015 Flush 0 Done Done Sent false
172282024-02-06T22:23:01.074Z 17 Acked 1016 Write 4 Done Done Sent false
172292024-02-06T22:23:01.074Z 18 Acked 1017 Write 5 Done Done Sent false
172302024-02-06T22:23:01.074Z 19 Acked 1018 Write 9 Done Done Sent false
172312024-02-06T22:23:01.074Z 20 Acked 1019 Read 7 Done Done Sent false
172322024-02-06T22:23:01.074Z 21 Acked 1020 Write 7 Done Done Sent false
172332024-02-06T22:23:01.074Z 22 Acked 1021 Read 10 Done Done Sent false
172342024-02-06T22:23:01.074Z 23 Acked 1022 Write 5 Done Done Sent false
172352024-02-06T22:23:01.074Z 24 Acked 1023 Write 3 Done Done Sent false
172362024-02-06T22:23:01.074Z 25 Acked 1024 Read 2 Done Done Sent false
172372024-02-06T22:23:01.074Z 26 Acked 1025 Write 10 Done Done Sent false
172382024-02-06T22:23:01.074Z 27 Acked 1026 Write 3 Done Done Sent false
172392024-02-06T22:23:01.074Z 28 Acked 1027 Write 6 Done Done Sent false
172402024-02-06T22:23:01.074Z 29 Acked 1028 Read 9 Done Done Sent false
172412024-02-06T22:23:01.074Z 30 Acked 1029 Write 8 Sent Sent Sent false
172422024-02-06T22:23:01.074Z STATES DS:0 DS:1 DS:2 TOTAL
172432024-02-06T22:23:01.074Z New 0 0 0 0
172442024-02-06T22:23:01.074Z Sent 1 1 16 18
172452024-02-06T22:23:01.074Z Done 29 29 14 72
172462024-02-06T22:23:01.074Z Skipped 0 0 0 0
172472024-02-06T22:23:01.075Z Error 0 0 0 0
172482024-02-06T22:23:01.075ZLast Flush: 1015 1015 0
172492024-02-06T22:23:01.075ZDownstairs last five completed:
172502024-02-06T22:23:01.075ZUpstairs last five completed: 30 29 28 27 26
172512024-02-06T22:23:01.075Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:00.247713798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1770}
172522024-02-06T22:23:01.075ZWrote out file "/tmp/test_repair_verify.data"
172532024-02-06T22:23:01.075Z
172542024-02-06T22:23:01.078ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172552024-02-06T22:23:01.096ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172562024-02-06T22:23:01.096Z 0 000-019 152 152 152 284 284 284 F F F
172572024-02-06T22:23:01.096Z 1 020-039 154 154 154 287 287 287 F F F
172582024-02-06T22:23:01.096Z 2 040-059 156 156 156 289 289 289 T T F <---
172592024-02-06T22:23:01.096Z 3 060-079 156 156 156 289 289 289 T T F <---
172602024-02-06T22:23:01.096Z 4 080-099 156 156 156 290 290 290 T T F <---
172612024-02-06T22:23:01.099Z 5 100-119 158 158 155 292 292 288 F F T <---
172622024-02-06T22:23:01.099Z 6 120-139 158 158 152 292 292 284 F F T <---
172632024-02-06T22:23:01.099Z 7 140-159 158 158 157 292 292 291 F F T <---
172642024-02-06T22:23:01.099Z 8 160-179 158 158 156 292 292 289 F F T <---
172652024-02-06T22:23:01.099Z 9 180-199 156 156 156 289 289 289 T T F <---
172662024-02-06T22:23:01.099Z 10 200-219 156 156 156 289 289 289 F F F
172672024-02-06T22:23:01.099Z 11 220-239 158 158 153 292 292 286 T T T <---
172682024-02-06T22:23:01.100Z 12 240-259 158 158 147 292 292 274 F F T <---
172692024-02-06T22:23:01.100Z 13 260-279 158 158 150 292 292 280 F F T <---
172702024-02-06T22:23:01.100Z 14 280-299 150 150 150 280 280 280 F F F
172712024-02-06T22:23:01.100Z 15 300-319 156 156 156 289 289 289 F F F
172722024-02-06T22:23:01.100Z 16 320-339 155 155 155 288 288 288 F F F
172732024-02-06T22:23:01.100Z 17 340-359 151 151 151 283 283 283 F F F
172742024-02-06T22:23:01.100Z 18 360-379 145 145 145 272 272 272 F F F
172752024-02-06T22:23:01.100Z 19 380-399 156 156 156 289 289 289 F F F
172762024-02-06T22:23:01.100Z 20 400-419 158 158 154 292 292 287 T T T <---
172772024-02-06T22:23:01.100Z 21 420-439 157 157 157 291 291 291 T T F <---
172782024-02-06T22:23:01.100Z 22 440-459 150 150 150 282 282 282 T T F <---
172792024-02-06T22:23:01.100Z 23 460-479 154 154 154 287 287 287 F F F
172802024-02-06T22:23:01.100Z 24 480-499 155 155 155 288 288 288 T T F <---
172812024-02-06T22:23:01.100Z 25 500-519 158 158 149 292 292 278 F F F <---
172822024-02-06T22:23:01.100Z 26 520-539 155 155 155 288 288 288 F F F
172832024-02-06T22:23:01.100Z 27 540-559 155 155 155 288 288 288 F F F
172842024-02-06T22:23:01.100Z 28 560-579 156 156 156 289 289 289 T T F <---
172852024-02-06T22:23:01.100Z 29 580-599 154 154 154 287 287 287 T T F <---
172862024-02-06T22:23:01.100ZMax gen: 158, Max flush: 292
172872024-02-06T22:23:01.100ZError: Difference in extent metadata found!
172882024-02-06T22:23:01.100ZA Difference in extent metadata is expected here
172892024-02-06T22:23:01.100ZOn loop 79
172902024-02-06T22:23:01.100Z
172912024-02-06T22:23:01.104ZVerifying data now
172922024-02-06T22:23:06.966ZNo change after verify
172932024-02-06T22:23:06.967ZLoop: 79 Downstairs dump after verify (and repair):
172942024-02-06T22:23:06.983ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172952024-02-06T22:23:06.983Z 0 000-019 152 152 152 284 284 284 F F F
172962024-02-06T22:23:06.983Z 1 020-039 154 154 154 287 287 287 F F F
172972024-02-06T22:23:06.983Z 2 040-059 159 159 159 293 293 293 F F F
172982024-02-06T22:23:06.983Z 3 060-079 159 159 159 293 293 293 F F F
172992024-02-06T22:23:06.983Z 4 080-099 159 159 159 293 293 293 F F F
173002024-02-06T22:23:06.983Z 5 100-119 158 158 158 292 292 292 F F F
173012024-02-06T22:23:06.990Z 6 120-139 158 158 158 292 292 292 F F F
173022024-02-06T22:23:06.990Z 7 140-159 158 158 158 292 292 292 F F F
173032024-02-06T22:23:06.990Z 8 160-179 158 158 158 292 292 292 F F F
173042024-02-06T22:23:06.990Z 9 180-199 159 159 159 293 293 293 F F F
173052024-02-06T22:23:06.990Z 10 200-219 156 156 156 289 289 289 F F F
173062024-02-06T22:23:06.990Z 11 220-239 159 159 159 293 293 293 F F F
173072024-02-06T22:23:06.990Z 12 240-259 158 158 158 292 292 292 F F F
173082024-02-06T22:23:06.990Z 13 260-279 158 158 158 292 292 292 F F F
173092024-02-06T22:23:06.990Z 14 280-299 150 150 150 280 280 280 F F F
173102024-02-06T22:23:06.990Z 15 300-319 156 156 156 289 289 289 F F F
173112024-02-06T22:23:06.991Z 16 320-339 155 155 155 288 288 288 F F F
173122024-02-06T22:23:06.991Z 17 340-359 151 151 151 283 283 283 F F F
173132024-02-06T22:23:06.991Z 18 360-379 145 145 145 272 272 272 F F F
173142024-02-06T22:23:06.991Z 19 380-399 156 156 156 289 289 289 F F F
173152024-02-06T22:23:06.991Z 20 400-419 159 159 159 293 293 293 F F F
173162024-02-06T22:23:06.991Z 21 420-439 159 159 159 293 293 293 F F F
173172024-02-06T22:23:06.991Z 22 440-459 159 159 159 293 293 293 F F F
173182024-02-06T22:23:06.991Z 23 460-479 154 154 154 287 287 287 F F F
173192024-02-06T22:23:06.991Z 24 480-499 159 159 159 293 293 293 F F F
173202024-02-06T22:23:06.991Z 25 500-519 158 158 158 292 292 292 F F F
173212024-02-06T22:23:06.991Z 26 520-539 155 155 155 288 288 288 F F F
173222024-02-06T22:23:06.991Z 27 540-559 155 155 155 288 288 288 F F F
173232024-02-06T22:23:06.991Z 28 560-579 159 159 159 293 293 293 F F F
173242024-02-06T22:23:06.991Z 29 580-599 159 159 159 293 293 293 F F F
173252024-02-06T22:23:06.991ZMax gen: 159, Max flush: 293
173262024-02-06T22:23:06.991Z
173272024-02-06T22:23:06.991ZBegin loop 80
173282024-02-06T22:23:06.991ZDownstairs to restart: 1
173292024-02-06T22:23:07.001Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180346256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778}
173302024-02-06T22:23:07.001Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180586603Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778}
173312024-02-06T22:23:07.001Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180612589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778}
173322024-02-06T22:23:07.012Z{"msg":"Crucible 4aad5426-9514-40b6-b30c-58a163ceee86 has session id: 1a7abc77-dc15-4606-bf70-f7fd38f50806","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
173332024-02-06T22:23:07.012Ztime":"2024-02-06T22:23:06.180645655Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173342024-02-06T22:23:07.012Z{"msg":"Upstairs opts: Upstairs UUID: 4aad5426-9514-40b6-b30c-58a163ceee86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180735793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173352024-02-06T22:23:07.012Z{"msg":"Crucible stats registered with UUID: 4aad5426-9514-40b6-b30c-58a163ceee86","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180769788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173362024-02-06T22:23:07.012Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180783986Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778}
173372024-02-06T22:23:07.012Z{"msg":"4aad5426-9514-40b6-b30c-58a163ceee86 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180901301Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173382024-02-06T22:23:07.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180926977Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173392024-02-06T22:23:07.012Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180942595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173402024-02-06T22:23:07.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180971631Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173412024-02-06T22:23:07.012Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.180995668Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173422024-02-06T22:23:07.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181020045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173432024-02-06T22:23:07.012Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.18105317Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173442024-02-06T22:23:07.012Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181094735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173452024-02-06T22:23:07.012Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.18112757Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173462024-02-06T22:23:07.012Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181153756Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173472024-02-06T22:23:07.012Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:06.18120045Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173482024-02-06T22:23:07.012Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:06.181234106Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173492024-02-06T22:23:07.013Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.18127375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173502024-02-06T22:23:07.013Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181301856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173512024-02-06T22:23:07.013Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:06.18135018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173522024-02-06T22:23:07.013Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181375417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173532024-02-06T22:23:07.013Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181398723Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173542024-02-06T22:23:07.013Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181555252Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173552024-02-06T22:23:07.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.181984394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173562024-02-06T22:23:07.013Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.182021419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173572024-02-06T22:23:07.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.182039277Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173582024-02-06T22:23:07.013Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.182072132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173592024-02-06T22:23:07.013Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.182639436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173602024-02-06T22:23:07.013Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.182677601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173612024-02-06T22:23:07.013Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.18319706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173622024-02-06T22:23:07.013Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.183233766Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173632024-02-06T22:23:07.013Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.183258372Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173642024-02-06T22:23:07.013Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.183285139Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173652024-02-06T22:23:07.013Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.183300446Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173662024-02-06T22:23:07.013Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:06.183314215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173672024-02-06T22:23:08.005Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.182916826Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173682024-02-06T22:23:08.013Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.183196128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173692024-02-06T22:23:08.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.183887215Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173702024-02-06T22:23:08.013Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.18392278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173712024-02-06T22:23:08.013Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.184390787Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173722024-02-06T22:23:08.014Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.184870362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173732024-02-06T22:23:08.014Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.184915286Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173742024-02-06T22:23:08.014Z{"msg":"[0]R flush_numbers[0..12]: [284, 287, 293, 293, 293, 292, 292, 292, 292, 293, 289, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.184935804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173752024-02-06T22:23:08.014Z{"msg":"[0]R generation[0..12]: [152, 154, 159, 159, 159, 158, 158, 158, 158, 159, 156, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.1849652Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173762024-02-06T22:23:08.014Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.184984767Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173772024-02-06T22:23:08.015Z{"msg":"[1]R flush_numbers[0..12]: [284, 287, 293, 293, 293, 292, 292, 292, 292, 293, 289, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185030841Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"Wait for a query_work_queue command to finish before sending IO
173782024-02-06T22:23:08.015Zdownstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173792024-02-06T22:23:08.015Z{"msg":"[1]R generation[0..12]: [152, 154, 159, 159, 159, 158, 158, 158, 158, 159, 156, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185054397Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173802024-02-06T22:23:08.015Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185084143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173812024-02-06T22:23:08.015Z{"msg":"[2]R flush_numbers[0..12]: [284, 287, 293, 293, 293, 292, 292, 292, 292, 293, 289, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185119259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173822024-02-06T22:23:08.016Z{"msg":"[2]R generation[0..12]: [152, 154, 159, 159, 159, 158, 158, 158, 158, 159, 156, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185138216Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173832024-02-06T22:23:08.016Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185166212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173842024-02-06T22:23:08.016Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.18518316Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173852024-02-06T22:23:08.016Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185208736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173862024-02-06T22:23:08.016Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185225674Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173872024-02-06T22:23:08.016Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185242492Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173882024-02-06T22:23:08.017Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.18526004Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173892024-02-06T22:23:08.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185275438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173902024-02-06T22:23:08.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185308163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173912024-02-06T22:23:08.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185327491Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173922024-02-06T22:23:08.018Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185341949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173932024-02-06T22:23:08.018Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185355657Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173942024-02-06T22:23:08.018Z{"msg":"4aad5426-9514-40b6-b30c-58a163ceee86 is now active with session: 1a7abc77-dc15-4606-bf70-f7fd38f50806","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185369145Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173952024-02-06T22:23:08.018Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185389232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778,"session_id":"1a7abc77-dc15-4606-bf70-f7fd38f50806"}
173962024-02-06T22:23:08.019Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:07.185402301Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778}
173972024-02-06T22:23:08.019ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
173982024-02-06T22:23:08.019ZLoading write count information from file "/tmp/test_repair_verify.data"
173992024-02-06T22:23:08.019ZRun Repair workload
174002024-02-06T22:23:08.020Z01/30 Write block 10 len 24576 data: 10 8 9 8 11 12
174012024-02-06T22:23:08.020Z02/30 Write block 571 len 32768 data: 18 18 17 18 14 14 15 15
174022024-02-06T22:23:08.020Z03/30 Write block 120 len 16384 data: 13 13 15 17
174032024-02-06T22:23:08.026Z04/30 Write block 398 len 16384 data: 9 10 11 14
174042024-02-06T22:23:08.026Z05/30 Write block 474 len 8192 data: 9 10
174052024-02-06T22:23:08.027Z06/30 Read block 379 len 28672
174062024-02-06T22:23:08.027Z07/30 Write block 133 len 12288 data: 15 13 11
174072024-02-06T22:23:08.027Z08/30 Read block 61 len 32768
174082024-02-06T22:23:08.027Z09/30 Write block 125 len 24576 data: 16 13 16 17 15 17
174092024-02-06T22:23:08.028Z10/30 Write block 352 len 16384 data: 22 20 18 18
174102024-02-06T22:23:08.028Z11/30 Write block 49 len 32768 data: 14 15 19 19 22 25 23 22
174112024-02-06T22:23:08.028Z12/30 Flush
174122024-02-06T22:23:08.028Z13/30 Write block 107 len 20480 data: 13 16 12 13 13
174132024-02-06T22:23:08.029Z14/30 Flush
174142024-02-06T22:23:08.029Z15/30 Write block 484 len 36864 data: 11 10 11 11 10 10 8 10 13
174152024-02-06T22:23:08.029Z16/30 Read block 475 len 8192
174162024-02-06T22:23:08.029Z17/30 Read block 147 len 28672
174172024-02-06T22:23:08.029Z18/30 Read block 580 len 24576
174182024-02-06T22:23:08.037Z19/30 Read block 203 len 28672
174192024-02-06T22:23:08.037Z20/30 Read block 75 len 28672
174202024-02-06T22:23:08.038Z21/30 Read block 76 len 32768
174212024-02-06T22:23:08.038Z22/30 Read block 491 len 36864
174222024-02-06T22:23:08.038Z23/30 Read block 567 len 20480
174232024-02-06T22:23:08.038Z24/30 Read block 86 len 24576
174242024-02-06T22:23:08.038Z25/30 Read block 377 len 36864
174252024-02-06T22:23:08.039Z26/30 Read block 455 len 40960
174262024-02-06T22:23:08.039Z27/30 Read block 181 len 8192
174272024-02-06T22:23:08.049Z28/30 Read block 8 len 20480
174282024-02-06T22:23:08.049Z29/30 Read block 212 len 32768
174292024-02-06T22:23:08.049Z30/30 Write block 193 len 28672 data: 11 14 12 15 12 9 9
174302024-02-06T22:23:08.049Z----------------------------------------------------------------
174312024-02-06T22:23:08.050Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
174322024-02-06T22:23:08.050ZStates: Active Active Active
174332024-02-06T22:23:08.050ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174342024-02-06T22:23:08.050Z 1 Acked 1000 Write 6 Done Sent Done false
174352024-02-06T22:23:08.050Z 2 Acked 1001 Write 8 Done Sent Done false
174362024-02-06T22:23:08.050Z 3 Acked 1002 Write 4 Done Sent Done false
174372024-02-06T22:23:08.050Z 4 Acked 1003 Write 4 Done Sent Done false
174382024-02-06T22:23:08.050Z 5 Acked 1004 Write 2 Done Sent Done false
174392024-02-06T22:23:08.051Z 6 Acked 1005 Read 7 Done Sent Done false
174402024-02-06T22:23:08.051Z 7 Acked 1006 Write 3 Done Sent Done false
174412024-02-06T22:23:08.051Z 8 Acked 1007 Read 8 Done Sent Done false
174422024-02-06T22:23:08.051Z 9 Acked 1008 Write 6 Done Sent Done false
174432024-02-06T22:23:08.051Z 10 Acked 1009 Write 4 Done Sent Done false
174442024-02-06T22:23:08.051Z 11 Acked 1010 Write 8 Done Sent Done false
174452024-02-06T22:23:08.051Z 12 Acked 1011 Flush 0 Done Sent Done false
174462024-02-06T22:23:08.051Z 13 Acked 1012 Write 5 Done Sent Done false
174472024-02-06T22:23:08.051Z 14 Acked 1013 Flush 0 Done Sent Done false
174482024-02-06T22:23:08.051Z 15 Acked 1014 Write 9 Done Sent Done false
174492024-02-06T22:23:08.051Z 16 Acked 1015 Read 2 Done Sent Done false
174502024-02-06T22:23:08.051Z 17 Acked 1016 Read 7 Done Sent Done false
174512024-02-06T22:23:08.051Z 18 Acked 1017 Read 6 Done Sent Done false
174522024-02-06T22:23:08.051Z 19 Acked 1018 Read 7 Done Sent Done false
174532024-02-06T22:23:08.051Z 20 Acked 1019 Read 7 Done Sent Done false
174542024-02-06T22:23:08.051Z 21 Acked 1020 Read 8 Done Sent Done false
174552024-02-06T22:23:08.052Z 22 Acked 1021 Read 9 Done Sent Done false
174562024-02-06T22:23:08.052Z 23 Acked 1022 Read 5 Done Sent Done false
174572024-02-06T22:23:08.052Z 24 Acked 1023 Read 6 Done Sent Done false
174582024-02-06T22:23:08.052Z 25 Acked 1024 Read 9 Done Sent Done false
174592024-02-06T22:23:08.052Z 26 Acked 1025 Read 10 Done Sent Done false
174602024-02-06T22:23:08.052Z 27 Acked 1026 Read 2 Done Sent Done false
174612024-02-06T22:23:08.052Z 28 Acked 1027 Read 5 Done Sent Done false
174622024-02-06T22:23:08.052Z 29 Acked 1028 Read 8 Done Sent Done false
174632024-02-06T22:23:08.052Z 30 Acked 1029 Write 7 Sent Sent Sent false
174642024-02-06T22:23:08.052Z STATES DS:0 DS:1 DS:2 TOTAL
174652024-02-06T22:23:08.052Z New 0 0 0 0
174662024-02-06T22:23:08.052Z Sent 1 30 1 32
174672024-02-06T22:23:08.052Z Done 29 0 29 58
174682024-02-06T22:23:08.052Z Skipped 0 0 0 0
174692024-02-06T22:23:08.052Z Error 0 0 0 0
174702024-02-06T22:23:08.053ZLast Flush: 1013 0 1013
174712024-02-06T22:23:08.053ZDownstairs last five completed:
174722024-02-06T22:23:08.053ZUpstairs last five completed: 30 29 28 27 26
174732024-02-06T22:23:08.053Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:07.216231585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1778}
174742024-02-06T22:23:08.053ZWrote out file "/tmp/test_repair_verify.data"
174752024-02-06T22:23:08.053Z
174762024-02-06T22:23:08.058ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174772024-02-06T22:23:08.062ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174782024-02-06T22:23:08.062Z 0 000-019 160 152 160 294 284 294 F F F <---
174792024-02-06T22:23:08.063Z 1 020-039 154 154 154 287 287 287 F F F
174802024-02-06T22:23:08.063Z 2 040-059 160 159 160 294 293 294 F F F <---
174812024-02-06T22:23:08.063Z 3 060-079 159 159 159 293 293 293 F F F
174822024-02-06T22:23:08.063Z 4 080-099 159 159 159 293 293 293 F F F
174832024-02-06T22:23:08.063Z 5 100-119 160 158 160 295 292 295 F F F <---
174842024-02-06T22:23:08.063Z 6 120-139 160 158 160 294 292 294 F F F <---
174852024-02-06T22:23:08.064Z 7 140-159 158 158 158 292 292 292 F F F
174862024-02-06T22:23:08.064Z 8 160-179 158 158 158 292 292 292 F F F
174872024-02-06T22:23:08.064Z 9 180-199 159 159 159 293 293 293 T F T <---
174882024-02-06T22:23:08.064Z 10 200-219 156 156 156 289 289 289 F F F
174892024-02-06T22:23:08.064Z 11 220-239 159 159 159 293 293 293 F F F
174902024-02-06T22:23:08.064Z 12 240-259 158 158 158 292 292 292 F F F
174912024-02-06T22:23:08.064Z 13 260-279 158 158 158 292 292 292 F F F
174922024-02-06T22:23:08.065Z 14 280-299 150 150 150 280 280 280 F F F
174932024-02-06T22:23:08.065Z 15 300-319 156 156 156 289 289 289 F F F
174942024-02-06T22:23:08.065Z 16 320-339 155 155 155 288 288 288 F F F
174952024-02-06T22:23:08.065Z 17 340-359 160 151 160 294 283 294 F F F <---
174962024-02-06T22:23:08.065Z 18 360-379 145 145 145 272 272 272 F F F
174972024-02-06T22:23:08.065Z 19 380-399 160 156 160 294 289 294 F F F <---
174982024-02-06T22:23:08.065Z 20 400-419 160 159 160 294 293 294 F F F <---
174992024-02-06T22:23:08.065Z 21 420-439 159 159 159 293 293 293 F F F
175002024-02-06T22:23:08.065Z 22 440-459 159 159 159 293 293 293 F F F
175012024-02-06T22:23:08.065Z 23 460-479 160 154 160 294 287 294 F F F <---
175022024-02-06T22:23:08.065Z 24 480-499 159 159 159 293 293 293 T F T <---
175032024-02-06T22:23:08.065Z 25 500-519 158 158 158 292 292 292 F F F
175042024-02-06T22:23:08.066Z 26 520-539 155 155 155 288 288 288 F F F
175052024-02-06T22:23:08.066Z 27 540-559 155 155 155 288 288 288 F F F
175062024-02-06T22:23:08.066Z 28 560-579 160 159 160 294 293 294 F F F <---
175072024-02-06T22:23:08.066Z 29 580-599 159 159 159 293 293 293 F F F
175082024-02-06T22:23:08.066ZMax gen: 160, Max flush: 295
175092024-02-06T22:23:08.066ZError: Difference in extent metadata found!
175102024-02-06T22:23:08.071ZA Difference in extent metadata is expected here
175112024-02-06T22:23:08.071ZOn loop 80
175122024-02-06T22:23:08.071Z
175132024-02-06T22:23:08.071ZVerifying data now
175142024-02-06T22:23:22.923ZNo change after verify
175152024-02-06T22:23:22.923ZLoop: 80 Downstairs dump after verify (and repair):
175162024-02-06T22:23:23.002ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175172024-02-06T22:23:23.002Z 0 000-019 160 160 160 294 294 294 F F F
175182024-02-06T22:23:23.003Z 1 020-039 154 154 154 287 287 287 F F F
175192024-02-06T22:23:23.003Z 2 040-059 160 160 160 294 294 294 F F F
175202024-02-06T22:23:23.003Z 3 060-079 159 159 159 293 293 293 F F F
175212024-02-06T22:23:23.003Z 4 080-099 159 159 159 293 293 293 F F F
175222024-02-06T22:23:23.003Z 5 100-119 160 160 160 295 295 295 F F F
175232024-02-06T22:23:23.003Z 6 120-139 160 160 160 294 294 294 F F F
175242024-02-06T22:23:23.003Z 7 140-159 158 158 158 292 292 292 F F F
175252024-02-06T22:23:23.003Z 8 160-179 158 158 158 292 292 292 F F F
175262024-02-06T22:23:23.003Z 9 180-199 161 161 161 296 296 296 F F F
175272024-02-06T22:23:23.003Z 10 200-219 156 156 156 289 289 289 F F F
175282024-02-06T22:23:23.003Z 11 220-239 159 159 159 293 293 293 F F F
175292024-02-06T22:23:23.003Z 12 240-259 158 158 158 292 292 292 F F F
175302024-02-06T22:23:23.003Z 13 260-279 158 158 158 292 292 292 F F F
175312024-02-06T22:23:23.003Z 14 280-299 150 150 150 280 280 280 F F F
175322024-02-06T22:23:23.003Z 15 300-319 156 156 156 289 289 289 F F F
175332024-02-06T22:23:23.003Z 16 320-339 155 155 155 288 288 288 F F F
175342024-02-06T22:23:23.003Z 17 340-359 160 160 160 294 294 294 F F F
175352024-02-06T22:23:23.003Z 18 360-379 145 145 145 272 272 272 F F F
175362024-02-06T22:23:23.004Z 19 380-399 160 160 160 294 294 294 F F F
175372024-02-06T22:23:23.004Z 20 400-419 160 160 160 294 294 294 F F F
175382024-02-06T22:23:23.004Z 21 420-439 159 159 159 293 293 293 F F F
175392024-02-06T22:23:23.004Z 22 440-459 159 159 159 293 293 293 F F F
175402024-02-06T22:23:23.004Z 23 460-479 160 160 160 294 294 294 F F F
175412024-02-06T22:23:23.004Z 24 480-499 161 161 161 296 296 296 F F F
175422024-02-06T22:23:23.004Z 25 500-519 158 158 158 292 292 292 F F F
175432024-02-06T22:23:23.004Z 26 520-539 155 155 155 288 288 288 F F F
175442024-02-06T22:23:23.004Z 27 540-559 155 155 155 288 288 288 F F F
175452024-02-06T22:23:23.004Z 28 560-579 160 160 160 294 294 294 F F F
175462024-02-06T22:23:23.004Z 29 580-599 159 159 159 293 293 293 F F F
175472024-02-06T22:23:23.004ZMax gen: 161, Max flush: 296
175482024-02-06T22:23:23.004Z
175492024-02-06T22:23:23.004ZBegin loop 81
175502024-02-06T22:23:23.004ZDownstairs to restart: 1
175512024-02-06T22:23:23.004Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827353997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786}
175522024-02-06T22:23:23.004Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
175532024-02-06T22:23:23.004Z,"time":"2024-02-06T22:23:11.827648797Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786}
175542024-02-06T22:23:23.004Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827691741Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786}
175552024-02-06T22:23:23.004Z{"msg":"Crucible 5b1924c3-63c6-4b21-af73-04eb481a70a9 has session id: 73dffb1b-86fc-4fe1-90ff-a3985c092af7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827733086Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175562024-02-06T22:23:23.004Z{"msg":"Upstairs opts: Upstairs UUID: 5b1924c3-63c6-4b21-af73-04eb481a70a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827763871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175572024-02-06T22:23:23.004Z{"msg":"Crucible stats registered with UUID: 5b1924c3-63c6-4b21-af73-04eb481a70a9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827787308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175582024-02-06T22:23:23.005Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827801126Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786}
175592024-02-06T22:23:23.005Z{"msg":"5b1924c3-63c6-4b21-af73-04eb481a70a9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827879906Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175602024-02-06T22:23:23.005Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827936228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175612024-02-06T22:23:23.005Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.827969644Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175622024-02-06T22:23:23.005Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.82799839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175632024-02-06T22:23:23.005Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828047963Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175642024-02-06T22:23:23.005Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828081979Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175652024-02-06T22:23:23.005Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828135451Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175662024-02-06T22:23:23.005Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828182785Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175672024-02-06T22:23:23.005Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828240037Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175682024-02-06T22:23:23.005Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.82829319Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175692024-02-06T22:23:23.005Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828342124Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175702024-02-06T22:23:23.005Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828446889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175712024-02-06T22:23:23.005Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828480165Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175722024-02-06T22:23:23.005Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.828988286Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175732024-02-06T22:23:23.005Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.829036959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175742024-02-06T22:23:23.005Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.829082763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175752024-02-06T22:23:23.005Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.829126367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175762024-02-06T22:23:23.005Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.829578057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175772024-02-06T22:23:23.005Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.82977388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175782024-02-06T22:23:23.006Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.830100196Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175792024-02-06T22:23:23.006Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.83013941Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175802024-02-06T22:23:23.006Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.830165407Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175812024-02-06T22:23:23.006Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.830189464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175822024-02-06T22:23:23.006Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.830231088Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175832024-02-06T22:23:23.006Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.830269833Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175842024-02-06T22:23:23.006Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.830370489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175852024-02-06T22:23:23.006Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.830404405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175862024-02-06T22:23:23.006Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.83088283Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175872024-02-06T22:23:23.006Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831318371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175882024-02-06T22:23:23.006Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831360836Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175892024-02-06T22:23:23.006Z{"msg":"[0]R flush_numbers[0..12]: [294, 287, 294, 293, 293, 295, 294, 292, 292, 296, 289, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831378493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175902024-02-06T22:23:23.006Z{"msg":"[0]R generation[0..12]: [160, 154, 160, 159, 159, 160, 160, 158, 158, 161, 156, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831394551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175912024-02-06T22:23:23.006Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831416838Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175922024-02-06T22:23:23.006Z{"msg":"[1]R flush_numbers[0..12]: [294, 287, 294, 293, 293, 295, 294, 292, 292, 296, 289, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831432626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175932024-02-06T22:23:23.006Z{"msg":"[1]R generation[0..12]: [160, 154, 160, 159, 159, 160, 160, 158, 158, 161, 156, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831453593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175942024-02-06T22:23:23.006Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831467801Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175952024-02-06T22:23:23.006Z{"msg":"[2]R flush_numbers[0..12]: [294, 287, 294, 293, 293, 295, 294, 292, 292, 296, 289, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831492078Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs",Wait for a query_work_queue command to finish before sending IO
175962024-02-06T22:23:23.006Z"":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175972024-02-06T22:23:23.006Z{"msg":"[2]R generation[0..12]: [160, 154, 160, 159, 159, 160, 160, 158, 158, 161, 156, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831510385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175982024-02-06T22:23:23.006Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831526143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
175992024-02-06T22:23:23.006Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.83154828Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176002024-02-06T22:23:23.007Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831562648Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176012024-02-06T22:23:23.007Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831590914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176022024-02-06T22:23:23.007Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831615691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176032024-02-06T22:23:23.007Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831651216Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176042024-02-06T22:23:23.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831670224Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176052024-02-06T22:23:23.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831686421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176062024-02-06T22:23:23.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831720517Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176072024-02-06T22:23:23.007Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831746264Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176082024-02-06T22:23:23.007Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831775029Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176092024-02-06T22:23:23.007Z{"msg":"5b1924c3-63c6-4b21-af73-04eb481a70a9 is now active with session: 73dffb1b-86fc-4fe1-90ff-a3985c092af7","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831793167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176102024-02-06T22:23:23.007Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831827013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786,"session_id":"73dffb1b-86fc-4fe1-90ff-a3985c092af7"}
176112024-02-06T22:23:23.007Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:11.831851579Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786}
176122024-02-06T22:23:23.007ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176132024-02-06T22:23:23.007ZLoading write count information from file "/tmp/test_repair_verify.data"
176142024-02-06T22:23:23.007ZRun Repair workload
176152024-02-06T22:23:23.007Z01/30 Write block 549 len 36864 data: 10 9 11 12 13 12 12 12 11
176162024-02-06T22:23:23.037Z02/30 Flush
176172024-02-06T22:23:23.037Z03/30 Write block 480 len 8192 data: 8 6
176182024-02-06T22:23:23.038Z04/30 Write block 56 len 28672 data: 23 18 16 16 16 12 9
176192024-02-06T22:23:23.038Z05/30 Read block 53 len 4096
176202024-02-06T22:23:23.038Z06/30 Read block 241 len 8192
176212024-02-06T22:23:23.038Z07/30 Write block 138 len 40960 data: 10 9 7 7 11 13 14 15 17 18
176222024-02-06T22:23:23.038Z08/30 Read block 35 len 24576
176232024-02-06T22:23:23.038Z09/30 Read block 47 len 12288
176242024-02-06T22:23:23.038Z10/30 Read block 460 len 36864
176252024-02-06T22:23:23.038Z11/30 Write block 269 len 28672 data: 15 13 12 12 9 7 10
176262024-02-06T22:23:23.038Z12/30 Read block 395 len 40960
176272024-02-06T22:23:23.038Z13/30 Write block 16 len 36864 data: 11 10 11 9 8 8 10 10 9
176282024-02-06T22:23:23.038Z14/30 Write block 214 len 12288 data: 17 14 11
176292024-02-06T22:23:23.038Z15/30 Read block 545 len 8192
176302024-02-06T22:23:23.038Z16/30 Read block 351 len 12288
176312024-02-06T22:23:23.038Z17/30 Read block 206 len 28672
176322024-02-06T22:23:23.038Z18/30 Read block 573 len 8192
176332024-02-06T22:23:23.038Z19/30 Read block 450 len 28672
176342024-02-06T22:23:23.038Z20/30 Read block 510 len 12288
176352024-02-06T22:23:23.039Z21/30 Read block 170 len 24576
176362024-02-06T22:23:23.039Z22/30 Write block 388 len 28672 data: 10 9 9 9 9 9 8
176372024-02-06T22:23:23.039Z23/30 Write block 272 len 16384 data: 13 10 8 11
176382024-02-06T22:23:23.039Z24/30 Write block 27 len 4096 data: 11
176392024-02-06T22:23:23.039Z25/30 Read block 419 len 8192
176402024-02-06T22:23:23.039Z26/30 Flush
176412024-02-06T22:23:23.039Z27/30 Write block 473 len 32768 data: 9 10 11 11 11 11 11 9
176422024-02-06T22:23:23.039Z28/30 Write block 30 len 40960 data: 12 9 7 8 8 7 8 9 11 10
176432024-02-06T22:23:23.039Z29/30 Read block 271 len 8192
176442024-02-06T22:23:23.039Z30/30 Write block 25 len 36864 data: 10 10 12 14 12 13 10 8 9
176452024-02-06T22:23:23.039Z----------------------------------------------------------------
176462024-02-06T22:23:23.039Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:28
176472024-02-06T22:23:23.039ZStates: Active Active Active
176482024-02-06T22:23:23.039ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176492024-02-06T22:23:23.039Z 3 Acked 1002 Write 2 Done Sent Done false
176502024-02-06T22:23:23.039Z 4 Acked 1003 Write 7 Done Sent Done false
176512024-02-06T22:23:23.039Z 5 Acked 1004 Read 1 Done Sent Done false
176522024-02-06T22:23:23.039Z 6 Acked 1005 Read 2 Done Sent Done false
176532024-02-06T22:23:23.039Z 7 Acked 1006 Write 10 Done Sent Done false
176542024-02-06T22:23:23.039Z 8 Acked 1007 Read 6 Done Sent Done false
176552024-02-06T22:23:23.040Z 9 Acked 1008 Read 3 Done Sent Done false
176562024-02-06T22:23:23.040Z 10 Acked 1009 Read 9 Done Sent Done false
176572024-02-06T22:23:23.040Z 11 Acked 1010 Write 7 Done Sent Done false
176582024-02-06T22:23:23.040Z 12 Acked 1011 Read 10 Done Sent Done false
176592024-02-06T22:23:23.040Z 13 Acked 1012 Write 9 Done Sent Done false
176602024-02-06T22:23:23.040Z 14 Acked 1013 Write 3 Done Sent Done false
176612024-02-06T22:23:23.040Z 15 Acked 1014 Read 2 Done Sent Done false
176622024-02-06T22:23:23.040Z 16 Acked 1015 Read 3 Done Sent Done false
176632024-02-06T22:23:23.040Z 17 Acked 1016 Read 7 Done Sent Done false
176642024-02-06T22:23:23.040Z 18 Acked 1017 Read 2 Done Sent Done false
176652024-02-06T22:23:23.040Z 19 Acked 1018 Read 7 Done Sent Done false
176662024-02-06T22:23:23.040Z 20 Acked 1019 Read 3 Done Sent Done false
176672024-02-06T22:23:23.040Z 21 Acked 1020 Read 6 Done Sent Done false
176682024-02-06T22:23:23.040Z 22 Acked 1021 Write 7 Done Sent Done false
176692024-02-06T22:23:23.040Z 23 Acked 1022 Write 4 Done Sent Done false
176702024-02-06T22:23:23.040Z 24 Acked 1023 Write 1 Done Sent Done false
176712024-02-06T22:23:23.040Z 25 Acked 1024 Read 2 Done Sent Done false
176722024-02-06T22:23:23.040Z 26 Acked 1025 Flush 0 Done Sent Done false
176732024-02-06T22:23:23.040Z 27 Acked 1026 Write 8 Done Sent Done false
176742024-02-06T22:23:23.040Z 28 Acked 1027 Write 10 Done Sent Done false
176752024-02-06T22:23:23.041Z 29 Acked 1028 Read 2 Done Sent Done false
176762024-02-06T22:23:23.041Z 30 Acked 1029 Write 9 Sent Sent Sent false
176772024-02-06T22:23:23.041Z STATES DS:0 DS:1 DS:2 TOTAL
176782024-02-06T22:23:23.041Z New 0 0 0 0
176792024-02-06T22:23:23.041Z Sent 1 28 1 30
176802024-02-06T22:23:23.041Z Done 27 0 27 54
176812024-02-06T22:23:23.041Z Skipped 0 0 0 0
176822024-02-06T22:23:23.041Z Error 0 0 0 0
176832024-02-06T22:23:23.041ZLast Flush: 1025 1001 1025
176842024-02-06T22:23:23.041ZDownstairs last five completed: 1001 1000
176852024-02-06T22:23:23.041ZUpstairs last five completed: 30 29 28 27 26
176862024-02-06T22:23:23.041Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:11.863090809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1786}
176872024-02-06T22:23:23.041ZWrote out file "/tmp/test_repair_verify.data"
176882024-02-06T22:23:23.041Z
176892024-02-06T22:23:23.041ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176902024-02-06T22:23:23.041ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176912024-02-06T22:23:23.041Z 0 000-019 162 160 162 298 294 298 F F F <---
176922024-02-06T22:23:23.041Z 1 020-039 162 154 162 298 287 298 T F T <---
176932024-02-06T22:23:23.041Z 2 040-059 162 160 162 298 294 298 F F F <---
176942024-02-06T22:23:23.041Z 3 060-079 162 159 162 298 293 298 F F F <---
176952024-02-06T22:23:23.041Z 4 080-099 159 159 159 293 293 293 F F F
176962024-02-06T22:23:23.041Z 5 100-119 160 160 160 295 295 295 F F F
176972024-02-06T22:23:23.041Z 6 120-139 162 160 162 298 294 298 F F F <---
176982024-02-06T22:23:23.041Z 7 140-159 162 158 162 298 292 298 F F F <---
176992024-02-06T22:23:23.042Z 8 160-179 158 158 158 292 292 292 F F F
177002024-02-06T22:23:23.042Z 9 180-199 161 161 161 296 296 296 F F F
177012024-02-06T22:23:23.042Z 10 200-219 162 156 162 298 289 298 F F F <---
177022024-02-06T22:23:23.042Z 11 220-239 159 159 159 293 293 293 F F F
177032024-02-06T22:23:23.042Z 12 240-259 158 158 158 292 292 292 F F F
177042024-02-06T22:23:23.042Z 13 260-279 162 158 162 298 292 298 F F F <---
177052024-02-06T22:23:23.042Z 14 280-299 150 150 150 280 280 280 F F F
177062024-02-06T22:23:23.042Z 15 300-319 156 156 156 289 289 289 F F F
177072024-02-06T22:23:23.042Z 16 320-339 155 155 155 288 288 288 F F F
177082024-02-06T22:23:23.042Z 17 340-359 160 160 160 294 294 294 F F F
177092024-02-06T22:23:23.042Z 18 360-379 145 145 145 272 272 272 F F F
177102024-02-06T22:23:23.042Z 19 380-399 162 160 162 298 294 298 F F F <---
177112024-02-06T22:23:23.042Z 20 400-419 160 160 160 294 294 294 F F F
177122024-02-06T22:23:23.042Z 21 420-439 159 159 159 293 293 293 F F F
177132024-02-06T22:23:23.042Z 22 440-459 159 159 159 293 293 293 F F F
177142024-02-06T22:23:23.042Z 23 460-479 160 160 160 294 294 294 T F T <---
177152024-02-06T22:23:23.042Z 24 480-499 162 161 162 298 296 298 T F T <---
177162024-02-06T22:23:23.050Z 25 500-519 158 158 158 292 292 292 F F F
177172024-02-06T22:23:23.050Z 26 520-539 155 155 155 288 288 288 F F F
177182024-02-06T22:23:23.050Z 27 540-559 162 162 162 297 297 297 F F F
177192024-02-06T22:23:23.050Z 28 560-579 160 160 160 294 294 294 F F F
177202024-02-06T22:23:23.050Z 29 580-599 159 159 159 293 293 293 F F F
177212024-02-06T22:23:23.050ZMax gen: 162, Max flush: 298
177222024-02-06T22:23:23.050ZError: Difference in extent metadata found!
177232024-02-06T22:23:23.050ZA Difference in extent metadata is expected here
177242024-02-06T22:23:23.051ZOn loop 81
177252024-02-06T22:23:23.051Z
177262024-02-06T22:23:23.051ZVerifying data now
177272024-02-06T22:23:23.051ZNo change after verify
177282024-02-06T22:23:23.051ZLoop: 81 Downstairs dump after verify (and repair):
177292024-02-06T22:23:23.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177302024-02-06T22:23:23.051Z 0 000-019 162 162 162 298 298 298 F F F
177312024-02-06T22:23:23.051Z 1 020-039 163 163 163 299 299 299 F F F
177322024-02-06T22:23:23.051Z 2 040-059 162 162 162 298 298 298 F F F
177332024-02-06T22:23:23.051Z 3 060-079 162 162 162 298 298 298 F F F
177342024-02-06T22:23:23.051Z 4 080-099 159 159 159 293 293 293 F F F
177352024-02-06T22:23:23.051Z 5 100-119 160 160 160 295 295 295 F F F
177362024-02-06T22:23:23.051Z 6 120-139 162 162 162 298 298 298 F F F
177372024-02-06T22:23:23.051Z 7 140-159 162 162 162 298 298 298 F F F
177382024-02-06T22:23:23.051Z 8 160-179 158 158 158 292 292 292 F F F
177392024-02-06T22:23:23.051Z 9 180-199 161 161 161 296 296 296 F F F
177402024-02-06T22:23:23.051Z 10 200-219 162 162 162 298 298 298 F F F
177412024-02-06T22:23:23.051Z 11 220-239 159 159 159 293 293 293 F F F
177422024-02-06T22:23:23.051Z 12 240-259 158 158 158 292 292 292 F F F
177432024-02-06T22:23:23.051Z 13 260-279 162 162 162 298 298 298 F F F
177442024-02-06T22:23:23.051Z 14 280-299 150 150 150 280 280 280 F F F
177452024-02-06T22:23:23.051Z 15 300-319 156 156 156 289 289 289 F F F
177462024-02-06T22:23:23.051Z 16 320-339 155 155 155 288 288 288 F F F
177472024-02-06T22:23:23.051Z 17 340-359 160 160 160 294 294 294 F F F
177482024-02-06T22:23:23.052Z 18 360-379 145 145 145 272 272 272 F F F
177492024-02-06T22:23:23.052Z 19 380-399 162 162 162 298 298 298 F F F
177502024-02-06T22:23:23.052Z 20 400-419 160 160 160 294 294 294 F F F
177512024-02-06T22:23:23.052Z 21 420-439 159 159 159 293 293 293 F F F
177522024-02-06T22:23:23.052Z 22 440-459 159 159 159 293 293 293 F F F
177532024-02-06T22:23:23.052Z 23 460-479 163 163 163 299 299 299 F F F
177542024-02-06T22:23:23.052Z 24 480-499 163 163 163 299 299 299 F F F
177552024-02-06T22:23:23.052Z 25 500-519 158 158 158 292 292 292 F F F
177562024-02-06T22:23:23.052Z 26 520-539 155 155 155 288 288 288 F F F
177572024-02-06T22:23:23.052Z 27 540-559 162 162 162 297 297 297 F F F
177582024-02-06T22:23:23.052Z 28 560-579 160 160 160 294 294 294 F F F
177592024-02-06T22:23:23.052Z 29 580-599 159 159 159 293 293 293 F F F
177602024-02-06T22:23:23.052ZMax gen: 163, Max flush: 299
177612024-02-06T22:23:23.052Z
177622024-02-06T22:23:23.052ZBegin loop 82
177632024-02-06T22:23:23.052ZDownstairs to restart: 1
177642024-02-06T22:23:23.052Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484070898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794}
177652024-02-06T22:23:23.052Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484344881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794}
177662024-02-06T22:23:23.052Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484391014Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794}
177672024-02-06T22:23:23.052ZCrucible runtime is spawned
177682024-02-06T22:23:23.053Z{"msg":"Crucible 90e356bf-4959-40bc-9ea0-9a5a5900cb83 has session id: 9c506f3e-5634-4959-994e-13af031ee58f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484443267Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177692024-02-06T22:23:23.053Z{"msg":"Upstairs opts: Upstairs UUID: 90e356bf-4959-40bc-9ea0-9a5a5900cb83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484488311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177702024-02-06T22:23:23.053Z{"msg":"Crucible stats registered with UUID: 90e356bf-4959-40bc-9ea0-9a5a5900cb83","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484549553Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177712024-02-06T22:23:23.053Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.48457569Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794}
177722024-02-06T22:23:23.053Z{"msg":"90e356bf-4959-40bc-9ea0-9a5a5900cb83 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484668327Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177732024-02-06T22:23:23.053Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484707722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177742024-02-06T22:23:23.053Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484743007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177752024-02-06T22:23:23.053Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484788051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177762024-02-06T22:23:23.053Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484836714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177772024-02-06T22:23:23.053Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484893976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177782024-02-06T22:23:23.053Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.48493759Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177792024-02-06T22:23:23.053Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.484980595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177802024-02-06T22:23:23.053Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485055355Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177812024-02-06T22:23:23.053Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485109937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177822024-02-06T22:23:23.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485155861Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177832024-02-06T22:23:23.053Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:16.485203765Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177842024-02-06T22:23:23.053Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:16.48523967Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177852024-02-06T22:23:23.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485256698Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177862024-02-06T22:23:23.053Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:16.485280024Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177872024-02-06T22:23:23.053Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485293402Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177882024-02-06T22:23:23.053Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485314799Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177892024-02-06T22:23:23.054Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485329088Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177902024-02-06T22:23:23.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485779057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177912024-02-06T22:23:23.054Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485818251Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177922024-02-06T22:23:23.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485835659Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177932024-02-06T22:23:23.054Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.485889752Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177942024-02-06T22:23:23.054Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.486436348Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177952024-02-06T22:23:23.054Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.486479732Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177962024-02-06T22:23:23.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.487091719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177972024-02-06T22:23:23.054Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.487126805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177982024-02-06T22:23:23.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.487142653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
177992024-02-06T22:23:23.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.487165969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178002024-02-06T22:23:23.054Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.487186997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178012024-02-06T22:23:23.054Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:16.487207294Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178022024-02-06T22:23:23.054Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.485618566Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178032024-02-06T22:23:23.054Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.485936193Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178042024-02-06T22:23:23.054Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.486654226Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178052024-02-06T22:23:23.054Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.4866972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178062024-02-06T22:23:23.054Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487198423Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178072024-02-06T22:23:23.054Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487684017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178082024-02-06T22:23:23.054Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487725281Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178092024-02-06T22:23:23.054Z{"msg":"[0]R flush_numbers[0..12]: [298, 299, 298, 298, 293, 295, 298, 298, 292, 296, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487745109Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178102024-02-06T22:23:23.054Z{"msg":"[0]R generation[0..12]: [162, 163, 162, 162, 159, 160, 162, 162, 158, 161, 162, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487764696Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178112024-02-06T22:23:23.055Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487792872Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178122024-02-06T22:23:23.055Z{"msg":"[1]R flush_numbers[0..12]: [298, 299, 298, 298, 293, 295, 298, 298, 292, 296, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487819688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178132024-02-06T22:23:23.055Z{"msg":"[1]R generation[0..12]: [162, 163, 162, 162, 159, 160, 162, 162, 158, 161, 162, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487846055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178142024-02-06T22:23:23.055ZWait for a query_work_queue command to finish before sending IO
178152024-02-06T22:23:23.055Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487875151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178162024-02-06T22:23:23.064Z{"msg":"[2]R flush_numbers[0..12]: [298, 299, 298, 298, 293, 295, 298, 298, 292, 296, 298, 293]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487902007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178172024-02-06T22:23:23.064Z{"msg":"[2]R generation[0..12]: [162, 163, 162, 162, 159, 160, 162, 162, 158, 161, 162, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487920735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178182024-02-06T22:23:23.064Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487946421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178192024-02-06T22:23:23.064Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487972498Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178202024-02-06T22:23:23.064Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.487998034Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178212024-02-06T22:23:23.064Z{"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488015362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178222024-02-06T22:23:23.064Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488040709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178232024-02-06T22:23:23.064Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488058476Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178242024-02-06T22:23:23.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488079713Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178252024-02-06T22:23:23.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488093511Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178262024-02-06T22:23:23.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488113489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178272024-02-06T22:23:23.064Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488133486Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178282024-02-06T22:23:23.064ZError: Difference in extent metadata found!
178292024-02-06T22:23:23.064Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488147234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178302024-02-06T22:23:23.064Z{"msg":"90e356bf-4959-40bc-9ea0-9a5a5900cb83 is now active with session: 9c506f3e-5634-4959-994e-13af031ee58f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488160273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178312024-02-06T22:23:23.064Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488172321Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794,"session_id":"9c506f3e-5634-4959-994e-13af031ee58f"}
178322024-02-06T22:23:23.064Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:17.488185119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794}
178332024-02-06T22:23:23.064ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178342024-02-06T22:23:23.064ZLoading write count information from file "/tmp/test_repair_verify.data"
178352024-02-06T22:23:23.065ZRun Repair workload
178362024-02-06T22:23:23.065Z01/30 Write block 482 len 36864 data: 8 9 12 11 12 12 11 11 9
178372024-02-06T22:23:23.065Z02/30 Read block 27 len 16384
178382024-02-06T22:23:23.065Z03/30 Read block 317 len 28672
178392024-02-06T22:23:23.065Z04/30 Read block 367 len 20480
178402024-02-06T22:23:23.065Z05/30 Write block 226 len 32768 data: 16 11 11 11 10 13 12 10
178412024-02-06T22:23:23.065Z06/30 Read block 446 len 28672
178422024-02-06T22:23:23.065Z07/30 Read block 144 len 24576
178432024-02-06T22:23:23.065Z08/30 Write block 280 len 24576 data: 15 14 12 9 10 8
178442024-02-06T22:23:23.065Z09/30 Read block 475 len 24576
178452024-02-06T22:23:23.065Z10/30 Read block 309 len 24576
178462024-02-06T22:23:23.065Z11/30 Read block 257 len 28672
178472024-02-06T22:23:23.065Z12/30 Write block 432 len 32768 data: 7 8 8 9 12 14 12 14
178482024-02-06T22:23:23.065Z13/30 Write block 65 len 20480 data: 10 11 10 10 10
178492024-02-06T22:23:23.065Z14/30 Read block 485 len 32768
178502024-02-06T22:23:23.065Z15/30 Read block 419 len 4096
178512024-02-06T22:23:23.065Z16/30 Read block 153 len 28672
178522024-02-06T22:23:23.065Z17/30 Write block 67 len 36864 data: 11 11 11 10 10 11 11 14 12
178532024-02-06T22:23:23.065Z18/30 Write block 452 len 24576 data: 9 10 15 14 10 10
178542024-02-06T22:23:23.065Z19/30 Write block 415 len 4096 data: 14
178552024-02-06T22:23:23.066Z20/30 Write block 278 len 20480 data: 14 15 16 15 13
178562024-02-06T22:23:23.066Z21/30 Write block 0 len 24576 data: 2 5 7 10 9 8
178572024-02-06T22:23:23.066Z22/30 Read block 282 len 36864
178582024-02-06T22:23:23.066Z23/30 Read block 267 len 8192
178592024-02-06T22:23:23.066Z24/30 Read block 392 len 24576
178602024-02-06T22:23:23.066Z25/30 Flush
178612024-02-06T22:23:23.066Z26/30 Write block 305 len 4096 data: 16
178622024-02-06T22:23:23.066Z27/30 Read block 419 len 20480
178632024-02-06T22:23:23.066Z28/30 Write block 47 len 16384 data: 12 13 15 16
178642024-02-06T22:23:23.066Z29/30 Write block 345 len 20480 data: 13 13 13 16 16
178652024-02-06T22:23:23.066Z30/30 Write block 66 len 12288 data: 12 12 12
178662024-02-06T22:23:23.066Z----------------------------------------------------------------
178672024-02-06T22:23:23.066Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
178682024-02-06T22:23:23.066ZStates: Active Active Active
178692024-02-06T22:23:23.066ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178702024-02-06T22:23:23.066Z 1 Acked 1000 Write 9 Done Done Done false
178712024-02-06T22:23:23.066Z 2 Acked 1001 Read 4 Done Done Done false
178722024-02-06T22:23:23.066Z 3 Acked 1002 Read 7 Done Sent Done false
178732024-02-06T22:23:23.066Z 4 Acked 1003 Read 5 Done Sent Done false
178742024-02-06T22:23:23.066Z 5 Acked 1004 Write 8 Done Sent Done false
178752024-02-06T22:23:23.066Z 6 Acked 1005 Read 7 Done Sent Done false
178762024-02-06T22:23:23.066Z 7 Acked 1006 Read 6 Done Sent Done false
178772024-02-06T22:23:23.067Z 8 Acked 1007 Write 6 Done Sent Done false
178782024-02-06T22:23:23.067Z 9 Acked 1008 Read 6 Done Sent Done false
178792024-02-06T22:23:23.067Z 10 Acked 1009 Read 6 Done Sent Done false
178802024-02-06T22:23:23.067Z 11 Acked 1010 Read 7 Done Sent Done false
178812024-02-06T22:23:23.067Z 12 Acked 1011 Write 8 Done Sent Done false
178822024-02-06T22:23:23.067Z 13 Acked 1012 Write 5 Done Sent Done false
178832024-02-06T22:23:23.067Z 14 Acked 1013 Read 8 Done Sent Done false
178842024-02-06T22:23:23.067Z 15 Acked 1014 Read 1 Done Sent Done false
178852024-02-06T22:23:23.067Z 16 Acked 1015 Read 7 Done Sent Done false
178862024-02-06T22:23:23.067Z 17 Acked 1016 Write 9 Done Sent Done false
178872024-02-06T22:23:23.067Z 18 Acked 1017 Write 6 Done Sent Done false
178882024-02-06T22:23:23.067Z 19 Acked 1018 Write 1 Done Sent Done false
178892024-02-06T22:23:23.067Z 20 Acked 1019 Write 5 Done Sent Done false
178902024-02-06T22:23:23.067Z 21 Acked 1020 Write 6 Done Sent Done false
178912024-02-06T22:23:23.067Z 22 Acked 1021 Read 9 Done Sent Done false
178922024-02-06T22:23:23.067Z 23 Acked 1022 Read 2 Done Sent Done false
178932024-02-06T22:23:23.067Z 24 Acked 1023 Read 6 Done Sent Done false
178942024-02-06T22:23:23.067Z 25 Acked 1024 Flush 0 Done Sent Done false
178952024-02-06T22:23:23.067Z 26 Acked 1025 Write 1 Done Sent Done false
178962024-02-06T22:23:23.067Z 27 Acked 1026 Read 5 Done Sent Done false
178972024-02-06T22:23:23.067Z 28 Acked 1027 Write 4 Done Sent Done false
178982024-02-06T22:23:23.067Z 29 Acked 1028 Write 5 Sent Sent Sent false
178992024-02-06T22:23:23.068Z 30 Acked 1029 Write 3 Sent Sent Sent false
179002024-02-06T22:23:23.068Z STATES DS:0 DS:1 DS:2 TOTAL
179012024-02-06T22:23:23.068Z New 0 0 0 0
179022024-02-06T22:23:23.068Z Sent 2 28 2 32
179032024-02-06T22:23:23.068Z Done 28 2 28 58
179042024-02-06T22:23:23.068Z Skipped 0 0 0 0
179052024-02-06T22:23:23.068Z Error 0 0 0 0
179062024-02-06T22:23:23.068ZLast Flush: 1024 0 1024
179072024-02-06T22:23:23.068ZDownstairs last five completed:
179082024-02-06T22:23:23.068ZUpstairs last five completed: 30 29 28 27 26
179092024-02-06T22:23:23.068Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:17.517813976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1794}
179102024-02-06T22:23:23.068ZWrote out file "/tmp/test_repair_verify.data"
179112024-02-06T22:23:23.068Z
179122024-02-06T22:23:23.068ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179132024-02-06T22:23:23.068ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179142024-02-06T22:23:23.068Z 0 000-019 164 162 164 300 298 300 F F F <---
179152024-02-06T22:23:23.068Z 1 020-039 163 163 163 299 299 299 F F F
179162024-02-06T22:23:23.166Z 2 040-059 162 162 162 298 298 298 T F T <---
179172024-02-06T22:23:23.166Z 3 060-079 164 162 164 300 298 300 T F T <---
179182024-02-06T22:23:23.166Z 4 080-099 159 159 159 293 293 293 F F F
179192024-02-06T22:23:23.166Z 5 100-119 160 160 160 295 295 295 F F F
179202024-02-06T22:23:23.167Z 6 120-139 162 162 162 298 298 298 F F F
179212024-02-06T22:23:23.167Z 7 140-159 162 162 162 298 298 298 F F F
179222024-02-06T22:23:23.167Z 8 160-179 158 158 158 292 292 292 F F F
179232024-02-06T22:23:23.167Z 9 180-199 161 161 161 296 296 296 F F F
179242024-02-06T22:23:23.167Z 10 200-219 162 162 162 298 298 298 F F F
179252024-02-06T22:23:23.167Z 11 220-239 164 159 164 300 293 300 F F F <---
179262024-02-06T22:23:23.167Z 12 240-259 158 158 158 292 292 292 F F F
179272024-02-06T22:23:23.167Z 13 260-279 164 162 164 300 298 300 F F F <---
179282024-02-06T22:23:23.167Z 14 280-299 164 150 164 300 280 300 F F F <---
179292024-02-06T22:23:23.167Z 15 300-319 156 156 156 289 289 289 T F T <---
179302024-02-06T22:23:23.167Z 16 320-339 155 155 155 288 288 288 F F F
179312024-02-06T22:23:23.167Z 17 340-359 160 160 160 294 294 294 T F T <---
179322024-02-06T22:23:23.167Z 18 360-379 145 145 145 272 272 272 F F F
179332024-02-06T22:23:23.168Z 19 380-399 162 162 162 298 298 298 F F F
179342024-02-06T22:23:23.168Z 20 400-419 164 160 164 300 294 300 F F F <---
179352024-02-06T22:23:23.168Z 21 420-439 164 159 164 300 293 300 F F F <---
179362024-02-06T22:23:23.168Z 22 440-459 164 159 164 300 293 300 F F F <---
179372024-02-06T22:23:23.168Z 23 460-479 163 163 163 299 299 299 F F F
179382024-02-06T22:23:23.168Z 24 480-499 164 163 164 300 299 300 F T F <---
179392024-02-06T22:23:23.168Z 25 500-519 158 158 158 292 292 292 F F F
179402024-02-06T22:23:23.168Z 26 520-539 155 155 155 288 288 288 F F F
179412024-02-06T22:23:23.168Z 27 540-559 162 162 162 297 297 297 F F F
179422024-02-06T22:23:23.168Z 28 560-579 160 160 160 294 294 294 F F F
179432024-02-06T22:23:23.168Z 29 580-599 159 159 159 293 293 293 F F F
179442024-02-06T22:23:23.168ZMax gen: 164, Max flush: 300
179452024-02-06T22:23:23.168ZA Difference in extent metadata is expected here
179462024-02-06T22:23:23.168ZOn loop 82
179472024-02-06T22:23:23.168Z
179482024-02-06T22:23:23.168ZVerifying data now
179492024-02-06T22:23:23.972ZNo change after verify
179502024-02-06T22:23:23.980ZLoop: 82 Downstairs dump after verify (and repair):
179512024-02-06T22:23:23.999ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179522024-02-06T22:23:23.999Z 0 000-019 164 164 164 300 300 300 F F F
179532024-02-06T22:23:23.999Z 1 020-039 163 163 163 299 299 299 F F F
179542024-02-06T22:23:23.999Z 2 040-059 165 165 165 301 301 301 F F F
179552024-02-06T22:23:23.999Z 3 060-079 165 165 165 301 301 301 F F F
179562024-02-06T22:23:23.999Z 4 080-099 159 159 159 293 293 293 F F F
179572024-02-06T22:23:23.999Z 5 100-119 160 160 160 295 295 295 F F F
179582024-02-06T22:23:23.999Z 6 120-139 162 162 162 298 298 298 F F F
179592024-02-06T22:23:23.999Z 7 140-159 162 162 162 298 298 298 F F F
179602024-02-06T22:23:23.999Z 8 160-179 158 158 158 292 292 292 F F F
179612024-02-06T22:23:23.999Z 9 180-199 161 161 161 296 296 296 F F F
179622024-02-06T22:23:23.999Z 10 200-219 162 162 162 298 298 298 F F F
179632024-02-06T22:23:23.999Z 11 220-239 164 164 164 300 300 300 F F F
179642024-02-06T22:23:23.999Z 12 240-259 158 158 158 292 292 292 F F F
179652024-02-06T22:23:23.999Z 13 260-279 164 164 164 300 300 300 F F F
179662024-02-06T22:23:23.999Z 14 280-299 164 164 164 300 300 300 F F F
179672024-02-06T22:23:23.999Z 15 300-319 165 165 165 301 301 301 F F F
179682024-02-06T22:23:24.000Z 16 320-339 155 155 155 288 288 288 F F F
179692024-02-06T22:23:24.000Z 17 340-359 165 165 165 301 301 301 F F F
179702024-02-06T22:23:24.000Z 18 360-379 145 145 145 272 272 272 F F F
179712024-02-06T22:23:24.000Z 19 380-399 162 162 162 298 298 298 F F F
179722024-02-06T22:23:24.000Z 20 400-419 164 164 164 300 300 300 F F F
179732024-02-06T22:23:24.000Z 21 420-439 164 164 164 300 300 300 F F F
179742024-02-06T22:23:24.000Z 22 440-459 164 164 164 300 300 300 F F F
179752024-02-06T22:23:24.000Z 23 460-479 163 163 163 299 299 299 F F F
179762024-02-06T22:23:24.000Z 24 480-499 164 164 164 300 300 300 F F F
179772024-02-06T22:23:24.000Z 25 500-519 158 158 158 292 292 292 F F F
179782024-02-06T22:23:24.000Z 26 520-539 155 155 155 288 288 288 F F F
179792024-02-06T22:23:24.000Z 27 540-559 162 162 162 297 297 297 F F F
179802024-02-06T22:23:24.000Z 28 560-579 160 160 160 294 294 294 F F F
179812024-02-06T22:23:24.000Z 29 580-599 159 159 159 293 293 293 F F F
179822024-02-06T22:23:24.000ZMax gen: 165, Max flush: 301
179832024-02-06T22:23:24.000Z
179842024-02-06T22:23:24.000ZBegin loop 83
179852024-02-06T22:23:24.000ZDownstairs to restart: 0
179862024-02-06T22:23:24.026Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186380093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802}
179872024-02-06T22:23:24.026Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186647207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal"Crucible runtime is spawned
179882024-02-06T22:23:24.026Z,"pid":1802}
179892024-02-06T22:23:24.026Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.18669863Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802}
179902024-02-06T22:23:24.026Z{"msg":"Crucible dd83a708-1448-44be-a6e5-3f5d9272bb6b has session id: 146d93be-3e0f-4dff-846a-02f6748aed93","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186752123Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179912024-02-06T22:23:24.026Z{"msg":"Upstairs opts: Upstairs UUID: dd83a708-1448-44be-a6e5-3f5d9272bb6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186807256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179922024-02-06T22:23:24.026Z{"msg":"Crucible stats registered with UUID: dd83a708-1448-44be-a6e5-3f5d9272bb6b","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186827333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179932024-02-06T22:23:24.026Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186844171Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802}
179942024-02-06T22:23:24.026Z{"msg":"dd83a708-1448-44be-a6e5-3f5d9272bb6b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186871807Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179952024-02-06T22:23:24.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186891904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179962024-02-06T22:23:24.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186910382Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179972024-02-06T22:23:24.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.186946737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179982024-02-06T22:23:24.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.18699585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
179992024-02-06T22:23:24.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187026896Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180002024-02-06T22:23:24.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187055132Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180012024-02-06T22:23:24.027Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187090578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180022024-02-06T22:23:24.027Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187125103Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180032024-02-06T22:23:24.027Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187173216Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180042024-02-06T22:23:24.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187210231Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180052024-02-06T22:23:24.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187258225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180062024-02-06T22:23:24.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.187303299Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180072024-02-06T22:23:24.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.188023701Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180082024-02-06T22:23:24.027Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.188061176Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180092024-02-06T22:23:24.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.188077814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180102024-02-06T22:23:24.027Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.188120788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180112024-02-06T22:23:24.027Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.188684212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180122024-02-06T22:23:24.027Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.188740424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180132024-02-06T22:23:24.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.18899698Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180142024-02-06T22:23:24.027Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189048473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180152024-02-06T22:23:24.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189301429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180162024-02-06T22:23:24.027Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189341863Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180172024-02-06T22:23:24.027Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189357671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180182024-02-06T22:23:24.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189371749Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180192024-02-06T22:23:24.027Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189386057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180202024-02-06T22:23:24.027Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189399535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180212024-02-06T22:23:24.027Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.189577131Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180222024-02-06T22:23:24.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190022221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180232024-02-06T22:23:24.027Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190053907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180242024-02-06T22:23:24.028Z{"msg":"[0]R flush_numbers[0..12]: [300, 299, 301, 301, 293, 295, 298, 298, 292, 296, 298, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190069275Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180252024-02-06T22:23:24.028Z{"msg":"[0]R generation[0..12]: [164, 163, 165, 165, 159, 160, 162, 162, 158, 161, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190090962Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180262024-02-06T22:23:24.028Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.19010597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180272024-02-06T22:23:24.028Z{"msg":"[1]R flush_numbers[0..12]: [300, 299, 301, 301, 293, 295, 298, 298, 292, 296, 298, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190129677Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180282024-02-06T22:23:24.028Z{"msg":"[1]R generation[0..12]: [164, 163, 165, 165, 159, 160, 162, 162, 158, 161, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190144075Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180292024-02-06T22:23:24.028Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190165442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180302024-02-06T22:23:24.028Z{"msg":"[2]R flush_numbers[0..12]: [300, 299, 301, 301, 293, 295, 298, 298, 292, 296, 298, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.1901801Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180312024-02-06T22:23:24.028Z{"msg":"[2]R generation[0..12]: [164, 163, 165, 165, 159, 160, 162, 162, 158, 161, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190194298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180322024-02-06T22:23:24.028Z{"msg"Wait for a query_work_queue command to finish before sending IO
180332024-02-06T22:23:24.028Z:"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190219355Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180342024-02-06T22:23:24.028Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190233063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180352024-02-06T22:23:24.028Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190246581Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180362024-02-06T22:23:24.028Z{"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190260209Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180372024-02-06T22:23:24.028Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190273307Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180382024-02-06T22:23:24.028Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190293535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180392024-02-06T22:23:24.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190306813Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180402024-02-06T22:23:24.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.19032805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180412024-02-06T22:23:24.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190341688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180422024-02-06T22:23:24.028Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190355736Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180432024-02-06T22:23:24.028Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190367925Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180442024-02-06T22:23:24.028Z{"msg":"dd83a708-1448-44be-a6e5-3f5d9272bb6b is now active with session: 146d93be-3e0f-4dff-846a-02f6748aed93","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190381133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180452024-02-06T22:23:24.028Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.19039999Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802,"session_id":"146d93be-3e0f-4dff-846a-02f6748aed93"}
180462024-02-06T22:23:24.028Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:23.190412838Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802}
180472024-02-06T22:23:24.028ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180482024-02-06T22:23:24.029ZLoading write count information from file "/tmp/test_repair_verify.data"
180492024-02-06T22:23:24.029ZRun Repair workload
180502024-02-06T22:23:24.029Z01/30 Write block 358 len 4096 data: 17
180512024-02-06T22:23:24.029Z02/30 Write block 331 len 32768 data: 7 8 11 14 15 16 16 16
180522024-02-06T22:23:24.029Z03/30 Write block 498 len 36864 data: 14 13 9 9 10 14 12 15 16
180532024-02-06T22:23:24.029Z04/30 Read block 506 len 12288
180542024-02-06T22:23:24.029Z05/30 Read block 583 len 16384
180552024-02-06T22:23:24.029Z06/30 Read block 6 len 20480
180562024-02-06T22:23:24.029Z07/30 Write block 81 len 40960 data: 8 11 11 10 10 11 13 12 12 10
180572024-02-06T22:23:24.029Z08/30 Write block 181 len 36864 data: 18 17 21 22 21 20 19 15 14
180582024-02-06T22:23:24.029Z09/30 Write block 60 len 20480 data: 17 13 10 9 7
180592024-02-06T22:23:24.041Z10/30 Read block 222 len 36864
180602024-02-06T22:23:24.041Z11/30 Read block 511 len 28672
180612024-02-06T22:23:24.041Z12/30 Write block 453 len 16384 data: 11 16 15 11
180622024-02-06T22:23:24.041Z13/30 Read block 505 len 4096
180632024-02-06T22:23:24.041Z14/30 Flush
180642024-02-06T22:23:24.041Z15/30 Write block 214 len 24576 data: 18 15 12 10 11 11
180652024-02-06T22:23:24.041Z16/30 Write block 246 len 40960 data: 10 14 14 16 13 10 10 10 12 10
180662024-02-06T22:23:24.041Z17/30 Read block 548 len 16384
180672024-02-06T22:23:24.041Z18/30 Read block 418 len 4096
180682024-02-06T22:23:24.041Z19/30 Write block 278 len 40960 data: 15 16 17 16 14 10 11 9 7 6
180692024-02-06T22:23:24.041Z20/30 Read block 258 len 20480
180702024-02-06T22:23:24.041Z21/30 Read block 553 len 8192
180712024-02-06T22:23:24.041Z22/30 Write block 278 len 8192 data: 16 17
180722024-02-06T22:23:24.041Z23/30 Write block 85 len 24576 data: 11 12 14 13 13 11
180732024-02-06T22:23:24.041Z24/30 Flush
180742024-02-06T22:23:24.041Z25/30 Write block 97 len 12288 data: 10 11 11
180752024-02-06T22:23:24.041Z26/30 Write block 445 len 16384 data: 9 9 7 9
180762024-02-06T22:23:24.051Z27/30 Write block 592 len 8192 data: 13 14
180772024-02-06T22:23:24.052Z28/30 Read block 401 len 8192
180782024-02-06T22:23:24.052Z29/30 Write block 347 len 20480 data: 14 17 17 16 21
180792024-02-06T22:23:24.052Z30/30 Read block 446 len 40960
180802024-02-06T22:23:24.052Z----------------------------------------------------------------
180812024-02-06T22:23:24.052Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
180822024-02-06T22:23:24.052ZStates: Active Active Active
180832024-02-06T22:23:24.052ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180842024-02-06T22:23:24.052Z 1 Acked 1000 Write 1 Done Done Done false
180852024-02-06T22:23:24.052Z 2 Acked 1001 Write 8 Sent Done Done false
180862024-02-06T22:23:24.052Z 3 Acked 1002 Write 9 Sent Done Done false
180872024-02-06T22:23:24.052Z 4 Acked 1003 Read 3 Sent Done Done false
180882024-02-06T22:23:24.052Z 5 Acked 1004 Read 4 Sent Done Done false
180892024-02-06T22:23:24.052Z 6 Acked 1005 Read 5 Sent Done Done false
180902024-02-06T22:23:24.052Z 7 Acked 1006 Write 10 Sent Done Done false
180912024-02-06T22:23:24.052Z 8 Acked 1007 Write 9 Sent Done Done false
180922024-02-06T22:23:24.053Z 9 Acked 1008 Write 5 Sent Done Done false
180932024-02-06T22:23:24.053Z 10 Acked 1009 Read 9 Sent Done Done false
180942024-02-06T22:23:24.053Z 11 Acked 1010 Read 7 Sent Done Done false
180952024-02-06T22:23:24.053Z 12 Acked 1011 Write 4 Sent Done Done false
180962024-02-06T22:23:24.053Z 13 Acked 1012 Read 1 Sent Done Done false
180972024-02-06T22:23:24.053Z 14 Acked 1013 Flush 0 Sent Done Done false
180982024-02-06T22:23:24.053Z 15 Acked 1014 Write 6 Sent Done Done false
180992024-02-06T22:23:24.053Z 16 Acked 1015 Write 10 Sent Done Done false
181002024-02-06T22:23:24.053Z 17 Acked 1016 Read 4 Sent Done Done false
181012024-02-06T22:23:24.053Z 18 Acked 1017 Read 1 Sent Done Done false
181022024-02-06T22:23:24.054Z 19 Acked 1018 Write 10 Sent Done Done false
181032024-02-06T22:23:24.054Z 20 Acked 1019 Read 5 Sent Done Done false
181042024-02-06T22:23:24.054Z 21 Acked 1020 Read 2 Sent Done Done false
181052024-02-06T22:23:24.054Z 22 Acked 1021 Write 2 Sent Done Done false
181062024-02-06T22:23:24.054Z 23 Acked 1022 Write 6 Sent Done Done false
181072024-02-06T22:23:24.054Z 24 Acked 1023 Flush 0 Sent Done Done false
181082024-02-06T22:23:24.054Z 25 Acked 1024 Write 3 Sent Done Done false
181092024-02-06T22:23:24.054Z 26 Acked 1025 Write 4 Sent Done Done false
181102024-02-06T22:23:24.054Z 27 Acked 1026 Write 2 Sent Done Done false
181112024-02-06T22:23:24.054Z 28 Acked 1027 Read 2 Sent Done Done false
181122024-02-06T22:23:24.054Z 29 Acked 1028 Write 5 Sent Done Done false
181132024-02-06T22:23:24.054Z 30 Acked 1029 Read 10 Sent Sent Done false
181142024-02-06T22:23:24.054Z STATES DS:0 DS:1 DS:2 TOTAL
181152024-02-06T22:23:24.054Z New 0 0 0 0
181162024-02-06T22:23:24.054Z Sent 29 1 0 30
181172024-02-06T22:23:24.054Z Done 1 29 30 60
181182024-02-06T22:23:24.054Z Skipped 0 0 0 0
181192024-02-06T22:23:24.054Z Error 0 0 0 0
181202024-02-06T22:23:24.055ZLast Flush: 0 1023 1023
181212024-02-06T22:23:24.055ZDownstairs last five completed:
181222024-02-06T22:23:24.055ZUpstairs last five completed: 30 29 28 27 26
181232024-02-06T22:23:24.055Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:23.221261501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1802}
181242024-02-06T22:23:24.055ZWrote out file "/tmp/test_repair_verify.data"
181252024-02-06T22:23:24.055Z
181262024-02-06T22:23:24.072ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181272024-02-06T22:23:24.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181282024-02-06T22:23:24.072Z 0 000-019 164 164 164 300 300 300 F F F
181292024-02-06T22:23:24.072Z 1 020-039 163 163 163 299 299 299 F F F
181302024-02-06T22:23:24.072Z 2 040-059 165 165 165 301 301 301 F F F
181312024-02-06T22:23:24.072Z 3 060-079 165 166 166 301 302 302 F F F <---
181322024-02-06T22:23:24.072Z 4 080-099 159 166 166 293 303 303 F T T <---
181332024-02-06T22:23:24.073Z 5 100-119 160 160 160 295 295 295 F F F
181342024-02-06T22:23:24.073Z 6 120-139 162 162 162 298 298 298 F F F
181352024-02-06T22:23:24.073Z 7 140-159 162 162 162 298 298 298 F F F
181362024-02-06T22:23:24.073Z 8 160-179 158 158 158 292 292 292 F F F
181372024-02-06T22:23:24.073Z 9 180-199 161 166 166 296 302 302 F F F <---
181382024-02-06T22:23:24.073Z 10 200-219 162 166 166 298 303 303 F F F <---
181392024-02-06T22:23:24.073Z 11 220-239 164 164 164 300 300 300 F F F
181402024-02-06T22:23:24.073Z 12 240-259 158 166 166 292 303 303 F F F <---
181412024-02-06T22:23:24.073Z 13 260-279 164 166 166 300 303 303 F F F <---
181422024-02-06T22:23:24.073Z 14 280-299 164 166 166 300 303 303 F F F <---
181432024-02-06T22:23:24.073Z 15 300-319 165 165 165 301 301 301 F F F
181442024-02-06T22:23:24.073Z 16 320-339 155 166 166 288 302 302 F F F <---
181452024-02-06T22:23:24.074Z 17 340-359 165 166 166 301 302 302 T T T <---
181462024-02-06T22:23:24.074Z 18 360-379 145 145 145 272 272 272 F F F
181472024-02-06T22:23:24.074Z 19 380-399 162 162 162 298 298 298 F F F
181482024-02-06T22:23:24.074Z 20 400-419 164 164 164 300 300 300 F F F
181492024-02-06T22:23:24.074Z 21 420-439 164 164 164 300 300 300 F F F
181502024-02-06T22:23:24.074Z 22 440-459 164 166 166 300 302 302 F T T <---
181512024-02-06T22:23:24.074Z 23 460-479 163 163 163 299 299 299 F F F
181522024-02-06T22:23:24.074Z 24 480-499 164 166 166 300 302 302 F F F <---
181532024-02-06T22:23:24.074Z 25 500-519 158 166 166 292 302 302 F F F <---
181542024-02-06T22:23:24.074Z 26 520-539 155 155 155 288 288 288 F F F
181552024-02-06T22:23:24.074Z 27 540-559 162 162 162 297 297 297 F F F
181562024-02-06T22:23:24.074Z 28 560-579 160 160 160 294 294 294 F F F
181572024-02-06T22:23:24.074Z 29 580-599 159 159 159 293 293 293 F T T <---
181582024-02-06T22:23:24.075ZMax gen: 166, Max flush: 303
181592024-02-06T22:23:24.075ZError: Difference in extent metadata found!
181602024-02-06T22:23:24.088ZA Difference in extent metadata is expected here
181612024-02-06T22:23:24.088ZOn loop 83
181622024-02-06T22:23:24.088Z
181632024-02-06T22:23:24.088ZVerifying data now
181642024-02-06T22:23:28.712ZNo change after verify
181652024-02-06T22:23:28.712ZLoop: 83 Downstairs dump after verify (and repair):
181662024-02-06T22:23:28.727ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181672024-02-06T22:23:28.727Z 0 000-019 164 164 164 300 300 300 F F F
181682024-02-06T22:23:28.727Z 1 020-039 163 163 163 299 299 299 F F F
181692024-02-06T22:23:28.727Z 2 040-059 165 165 165 301 301 301 F F F
181702024-02-06T22:23:28.727Z 3 060-079 166 166 166 302 302 302 F F F
181712024-02-06T22:23:28.727Z 4 080-099 167 167 167 304 304 304 F F F
181722024-02-06T22:23:28.727Z 5 100-119 160 160 160 295 295 295 F F F
181732024-02-06T22:23:28.727Z 6 120-139 162 162 162 298 298 298 F F F
181742024-02-06T22:23:28.728Z 7 140-159 162 162 162 298 298 298 F F F
181752024-02-06T22:23:28.728Z 8 160-179 158 158 158 292 292 292 F F F
181762024-02-06T22:23:28.728Z 9 180-199 166 166 166 302 302 302 F F F
181772024-02-06T22:23:28.728Z 10 200-219 166 166 166 303 303 303 F F F
181782024-02-06T22:23:28.728Z 11 220-239 164 164 164 300 300 300 F F F
181792024-02-06T22:23:28.728Z 12 240-259 166 166 166 303 303 303 F F F
181802024-02-06T22:23:28.728Z 13 260-279 166 166 166 303 303 303 F F F
181812024-02-06T22:23:28.728Z 14 280-299 166 166 166 303 303 303 F F F
181822024-02-06T22:23:28.731Z 15 300-319 165 165 165 301 301 301 F F F
181832024-02-06T22:23:28.731Z 16 320-339 166 166 166 302 302 302 F F F
181842024-02-06T22:23:28.731Z 17 340-359 167 167 167 304 304 304 F F F
181852024-02-06T22:23:28.731Z 18 360-379 145 145 145 272 272 272 F F F
181862024-02-06T22:23:28.731Z 19 380-399 162 162 162 298 298 298 F F F
181872024-02-06T22:23:28.731Z 20 400-419 164 164 164 300 300 300 F F F
181882024-02-06T22:23:28.731Z 21 420-439 164 164 164 300 300 300 F F F
181892024-02-06T22:23:28.731Z 22 440-459 167 167 167 304 304 304 F F F
181902024-02-06T22:23:28.731Z 23 460-479 163 163 163 299 299 299 F F F
181912024-02-06T22:23:28.732Z 24 480-499 166 166 166 302 302 302 F F F
181922024-02-06T22:23:28.732Z 25 500-519 166 166 166 302 302 302 F F F
181932024-02-06T22:23:28.732Z 26 520-539 155 155 155 288 288 288 F F F
181942024-02-06T22:23:28.732Z 27 540-559 162 162 162 297 297 297 F F F
181952024-02-06T22:23:28.732Z 28 560-579 160 160 160 294 294 294 F F F
181962024-02-06T22:23:28.732Z 29 580-599 167 167 167 304 304 304 F F F
181972024-02-06T22:23:28.732ZMax gen: 167, Max flush: 304
181982024-02-06T22:23:28.732Z
181992024-02-06T22:23:28.732ZBegin loop 84
182002024-02-06T22:23:28.732ZDownstairs to restart: 2
182012024-02-06T22:23:28.749Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.925569671Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810}
182022024-02-06T22:23:28.749Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.925820517Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810}
182032024-02-06T22:23:28.750Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.925838464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810}
182042024-02-06T22:23:28.750Z{"msg":Crucible runtime is spawned
182052024-02-06T22:23:28.755Z"Crucible eb89f005-8dd7-4dcd-ba76-d6413038838f has session id: c200dc5e-f251-4614-b715-2c8e8d214238","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.925868431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182062024-02-06T22:23:28.755Z{"msg":"Upstairs opts: Upstairs UUID: eb89f005-8dd7-4dcd-ba76-d6413038838f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.925923753Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182072024-02-06T22:23:28.755Z{"msg":"Crucible stats registered with UUID: eb89f005-8dd7-4dcd-ba76-d6413038838f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.925962728Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182082024-02-06T22:23:28.755Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.925988544Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810}
182092024-02-06T22:23:28.755Z{"msg":"eb89f005-8dd7-4dcd-ba76-d6413038838f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926160611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182102024-02-06T22:23:28.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926189857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182112024-02-06T22:23:28.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926206285Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182122024-02-06T22:23:28.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926235611Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182132024-02-06T22:23:28.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926271156Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182142024-02-06T22:23:28.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926293123Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182152024-02-06T22:23:28.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926326198Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182162024-02-06T22:23:28.755Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926340097Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182172024-02-06T22:23:28.755Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926375902Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182182024-02-06T22:23:28.755Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926402938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182192024-02-06T22:23:28.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926440433Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182202024-02-06T22:23:28.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.92646659Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182212024-02-06T22:23:28.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.926575335Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182222024-02-06T22:23:28.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.927173444Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182232024-02-06T22:23:28.755Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.927216038Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182242024-02-06T22:23:28.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.927232806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182252024-02-06T22:23:28.755Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.927248094Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182262024-02-06T22:23:28.756Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.927841504Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182272024-02-06T22:23:28.756Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.927887438Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182282024-02-06T22:23:28.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.928344786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182292024-02-06T22:23:28.756Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.928396619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182302024-02-06T22:23:28.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.92853625Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182312024-02-06T22:23:28.756Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.928586663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182322024-02-06T22:23:28.756Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.928632177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182332024-02-06T22:23:28.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.928655994Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182342024-02-06T22:23:28.756Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.928671012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182352024-02-06T22:23:28.756Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.92868472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182362024-02-06T22:23:28.756Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.928883043Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182372024-02-06T22:23:28.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929315535Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182382024-02-06T22:23:28.756Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.92935137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182392024-02-06T22:23:28.756Z{"msg":"[0]R flush_numbers[0..12]: [300, 299, 301, 302, 304, 295, 298, 298, 292, 302, 303, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929367278Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182402024-02-06T22:23:28.756Z{"msg":"[0]R generation[0..12]: [164, 163, 165, 166, 167, 160, 162, 162, 158, 166, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929382106Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182412024-02-06T22:23:28.756Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929404843Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182422024-02-06T22:23:28.756Z{"msg":"[1]R flush_numbers[0..12]: [300, 299, 301, 302, 304, 295, 298, 298, 292, 302, 303, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.92941992Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182432024-02-06T22:23:28.756Z{"msg":"[1]R generation[0..12]: [164, 163, 165, 166, 167, 160, 162, 162, 158, 166, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929434019Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182442024-02-06T22:23:28.756Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929455946Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182452024-02-06T22:23:28.756Z{"msg":"[2]R flush_numbers[0..12]: [300, 299, 301, 302, 304, 295, 298, 298, 292, 302, 303, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929470854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182462024-02-06T22:23:28.756Z{"msg":"[2]R generation[0..12]: [164, 163, 165, 166, 167, 160, 162, 162, 158, 166, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929484722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182472024-02-06T22:23:28.756Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929507159Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182482024-02-06T22:23:28.760Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929520657Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182492024-02-06T22:23:28.760Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929546814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182502024-02-06T22:23:28.761Z{"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929560912Z","hostname":"ip-10-150-1-27.us-west-2.compute.internalWait for a query_work_queue command to finish before sending IO
182512024-02-06T22:23:28.761Z","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182522024-02-06T22:23:28.761Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929577549Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182532024-02-06T22:23:28.761Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929591017Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182542024-02-06T22:23:28.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929603976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182552024-02-06T22:23:28.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929617304Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182562024-02-06T22:23:28.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.92964694Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182572024-02-06T22:23:28.761Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929679965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182582024-02-06T22:23:28.761Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929694963Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182592024-02-06T22:23:28.761Z{"msg":"eb89f005-8dd7-4dcd-ba76-d6413038838f is now active with session: c200dc5e-f251-4614-b715-2c8e8d214238","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929726809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182602024-02-06T22:23:28.761Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929757795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810,"session_id":"c200dc5e-f251-4614-b715-2c8e8d214238"}
182612024-02-06T22:23:28.761Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:27.929773433Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810}
182622024-02-06T22:23:28.761ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182632024-02-06T22:23:28.761ZLoading write count information from file "/tmp/test_repair_verify.data"
182642024-02-06T22:23:28.761ZRun Repair workload
182652024-02-06T22:23:28.761Z01/30 Write block 201 len 20480 data: 11 15 14 15 15
182662024-02-06T22:23:28.761Z02/30 Read block 10 len 20480
182672024-02-06T22:23:28.761Z03/30 Read block 0 len 24576
182682024-02-06T22:23:28.761Z04/30 Read block 433 len 36864
182692024-02-06T22:23:28.761Z05/30 Write block 51 len 32768 data: 20 20 23 26 24 24 19 17
182702024-02-06T22:23:28.761Z06/30 Write block 54 len 4096 data: 27
182712024-02-06T22:23:28.765Z07/30 Flush
182722024-02-06T22:23:28.765Z08/30 Write block 402 len 20480 data: 15 15 14 15 16
182732024-02-06T22:23:28.765Z09/30 Flush
182742024-02-06T22:23:28.768Z10/30 Write block 334 len 40960 data: 15 16 17 17 17 16 16 16 20 15
182752024-02-06T22:23:28.768Z11/30 Write block 95 len 12288 data: 9 9 11
182762024-02-06T22:23:28.768Z12/30 Write block 561 len 28672 data: 10 11 9 11 13 13 14
182772024-02-06T22:23:28.768Z13/30 Write block 578 len 16384 data: 16 16 15 13
182782024-02-06T22:23:28.768Z14/30 Read block 37 len 4096
182792024-02-06T22:23:28.771Z15/30 Read block 445 len 32768
182802024-02-06T22:23:28.771Z16/30 Write block 449 len 28672 data: 9 9 10 10 12 17 16
182812024-02-06T22:23:28.771Z17/30 Write block 135 len 36864 data: 12 10 9 11 10 8 8 12 14
182822024-02-06T22:23:28.774Z18/30 Write block 247 len 8192 data: 15 15
182832024-02-06T22:23:28.774Z19/30 Write block 157 len 28672 data: 18 14 16 14 14 18 15
182842024-02-06T22:23:28.774Z20/30 Read block 443 len 24576
182852024-02-06T22:23:28.774Z21/30 Read block 400 len 28672
182862024-02-06T22:23:28.777Z22/30 Read block 399 len 28672
182872024-02-06T22:23:28.777Z23/30 Read block 242 len 20480
182882024-02-06T22:23:28.778Z24/30 Write block 381 len 36864 data: 13 11 11 11 12 14 10 11 10
182892024-02-06T22:23:28.778Z25/30 Read block 580 len 16384
182902024-02-06T22:23:28.780Z26/30 Read block 352 len 24576
182912024-02-06T22:23:28.781Z27/30 Write block 495 len 20480 data: 13 15 14 15 14
182922024-02-06T22:23:28.783Z28/30 Write block 556 len 36864 data: 13 12 13 10 9 11 12 10 12
182932024-02-06T22:23:28.783Z29/30 Read block 470 len 28672
182942024-02-06T22:23:28.790Z30/30 Write block 257 len 36864 data: 10 12 13 14 19 20 21 19 21
182952024-02-06T22:23:28.790Z----------------------------------------------------------------
182962024-02-06T22:23:28.790Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
182972024-02-06T22:23:28.790ZStates: Active Active Active
182982024-02-06T22:23:28.790ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182992024-02-06T22:23:28.790Z 1 Acked 1000 Write 5 Done Done Done false
183002024-02-06T22:23:28.790Z 2 Acked 1001 Read 5 Done Done Done false
183012024-02-06T22:23:28.790Z 3 Acked 1002 Read 6 Done Done Sent false
183022024-02-06T22:23:28.790Z 4 Acked 1003 Read 9 Done Done Sent false
183032024-02-06T22:23:28.790Z 5 Acked 1004 Write 8 Done Done Sent false
183042024-02-06T22:23:28.790Z 6 Acked 1005 Write 1 Done Done Sent false
183052024-02-06T22:23:28.790Z 7 Acked 1006 Flush 0 Done Done Sent false
183062024-02-06T22:23:28.790Z 8 Acked 1007 Write 5 Done Done Sent false
183072024-02-06T22:23:28.790Z 9 Acked 1008 Flush 0 Done Done Sent false
183082024-02-06T22:23:28.790Z 10 Acked 1009 Write 10 Done Done Sent false
183092024-02-06T22:23:28.790Z 11 Acked 1010 Write 3 Done Done Sent false
183102024-02-06T22:23:28.790Z 12 Acked 1011 Write 7 Done Done Sent false
183112024-02-06T22:23:28.790Z 13 Acked 1012 Write 4 Done Done Sent false
183122024-02-06T22:23:28.790Z 14 Acked 1013 Read 1 Done Done Sent false
183132024-02-06T22:23:28.790Z 15 Acked 1014 Read 8 Done Done Sent false
183142024-02-06T22:23:28.790Z 16 Acked 1015 Write 7 Done Done Sent false
183152024-02-06T22:23:28.790Z 17 Acked 1016 Write 9 Done Done Sent false
183162024-02-06T22:23:28.791Z 18 Acked 1017 Write 2 Done Done Sent false
183172024-02-06T22:23:28.791Z 19 Acked 1018 Write 7 Done Done Sent false
183182024-02-06T22:23:28.791Z 20 Acked 1019 Read 6 Done Done Sent false
183192024-02-06T22:23:28.791Z 21 Acked 1020 Read 7 Done Done Sent false
183202024-02-06T22:23:28.791Z 22 Acked 1021 Read 7 Done Done Sent false
183212024-02-06T22:23:28.791Z 23 Acked 1022 Read 5 Done Done Sent false
183222024-02-06T22:23:28.791Z 24 Acked 1023 Write 9 Done Done Sent false
183232024-02-06T22:23:28.791Z 25 Acked 1024 Read 4 Done Done Sent false
183242024-02-06T22:23:28.791Z 26 Acked 1025 Read 6 Done Done Sent false
183252024-02-06T22:23:28.791Z 27 Acked 1026 Write 5 Done Done Sent false
183262024-02-06T22:23:28.791Z 28 Acked 1027 Write 9 Done Done Sent false
183272024-02-06T22:23:28.791Z 29 Acked 1028 Read 7 Done Done Sent false
183282024-02-06T22:23:28.791Z 30 Acked 1029 Write 9 Sent Sent Sent false
183292024-02-06T22:23:28.791Z STATES DS:0 DS:1 DS:2 TOTAL
183302024-02-06T22:23:28.791Z New 0 0 0 0
183312024-02-06T22:23:28.791Z Sent 1 1 28 30
183322024-02-06T22:23:28.791Z Done 29 29 2 60
183332024-02-06T22:23:28.791Z Skipped 0 0 0 0
183342024-02-06T22:23:28.791Z Error 0 0 0 0
183352024-02-06T22:23:28.791ZLast Flush: 1008 1008 0
183362024-02-06T22:23:28.791ZDownstairs last five completed:
183372024-02-06T22:23:28.791ZUpstairs last five completed: 30 29 28 27 26
183382024-02-06T22:23:28.791Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:27.96073323Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1810}
183392024-02-06T22:23:28.791ZWrote out file "/tmp/test_repair_verify.data"
183402024-02-06T22:23:28.795Z
183412024-02-06T22:23:28.795ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183422024-02-06T22:23:28.812ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183432024-02-06T22:23:28.812Z 0 000-019 164 164 164 300 300 300 F F F
183442024-02-06T22:23:28.812Z 1 020-039 163 163 163 299 299 299 F F F
183452024-02-06T22:23:28.812Z 2 040-059 168 168 165 305 305 301 F F F <---
183462024-02-06T22:23:28.812Z 3 060-079 166 166 166 302 302 302 F F F
183472024-02-06T22:23:28.812Z 4 080-099 167 167 167 304 304 304 T T F <---
183482024-02-06T22:23:28.812Z 5 100-119 160 160 160 295 295 295 F F F
183492024-02-06T22:23:28.812Z 6 120-139 162 162 162 298 298 298 T T F <---
183502024-02-06T22:23:28.812Z 7 140-159 162 162 162 298 298 298 T T F <---
183512024-02-06T22:23:28.812Z 8 160-179 158 158 158 292 292 292 T T F <---
183522024-02-06T22:23:28.812Z 9 180-199 166 166 166 302 302 302 F F F
183532024-02-06T22:23:28.812Z 10 200-219 168 168 166 305 305 303 F F T <---
183542024-02-06T22:23:28.812Z 11 220-239 164 164 164 300 300 300 F F F
183552024-02-06T22:23:28.812Z 12 240-259 166 166 166 303 303 303 T T F <---
183562024-02-06T22:23:28.812Z 13 260-279 166 166 166 303 303 303 T T F <---
183572024-02-06T22:23:28.815Z 14 280-299 166 166 166 303 303 303 F F F
183582024-02-06T22:23:28.816Z 15 300-319 165 165 165 301 301 301 F F F
183592024-02-06T22:23:28.816Z 16 320-339 166 166 166 302 302 302 T T F <---
183602024-02-06T22:23:28.816Z 17 340-359 167 167 167 304 304 304 T T F <---
183612024-02-06T22:23:28.816Z 18 360-379 145 145 145 272 272 272 F F F
183622024-02-06T22:23:28.816Z 19 380-399 162 162 162 298 298 298 T T F <---
183632024-02-06T22:23:28.816Z 20 400-419 168 168 164 306 306 300 F F F <---
183642024-02-06T22:23:28.816Z 21 420-439 164 164 164 300 300 300 F F F
183652024-02-06T22:23:28.816Z 22 440-459 167 167 167 304 304 304 T T F <---
183662024-02-06T22:23:28.816Z 23 460-479 163 163 163 299 299 299 F F F
183672024-02-06T22:23:28.816Z 24 480-499 166 166 166 302 302 302 T T F <---
183682024-02-06T22:23:28.816Z 25 500-519 166 166 166 302 302 302 F F F
183692024-02-06T22:23:28.816Z 26 520-539 155 155 155 288 288 288 F F F
183702024-02-06T22:23:28.816Z 27 540-559 162 162 162 297 297 297 T T F <---
183712024-02-06T22:23:28.816Z 28 560-579 160 160 160 294 294 294 T T F <---
183722024-02-06T22:23:28.816Z 29 580-599 167 167 167 304 304 304 T T F <---
183732024-02-06T22:23:28.816ZMax gen: 168, Max flush: 306
183742024-02-06T22:23:28.816ZError: Difference in extent metadata found!
183752024-02-06T22:23:28.816ZA Difference in extent metadata is expected here
183762024-02-06T22:23:28.816ZOn loop 84
183772024-02-06T22:23:28.816Z
183782024-02-06T22:23:28.820ZVerifying data now
183792024-02-06T22:23:33.638ZNo change after verify
183802024-02-06T22:23:33.638ZLoop: 84 Downstairs dump after verify (and repair):
183812024-02-06T22:23:33.653ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183822024-02-06T22:23:33.653Z 0 000-019 164 164 164 300 300 300 F F F
183832024-02-06T22:23:33.653Z 1 020-039 163 163 163 299 299 299 F F F
183842024-02-06T22:23:33.653Z 2 040-059 168 168 168 305 305 305 F F F
183852024-02-06T22:23:33.653Z 3 060-079 166 166 166 302 302 302 F F F
183862024-02-06T22:23:33.653Z 4 080-099 169 169 169 307 307 307 F F F
183872024-02-06T22:23:33.656Z 5 100-119 160 160 160 295 295 295 F F F
183882024-02-06T22:23:33.656Z 6 120-139 169 169 169 307 307 307 F F F
183892024-02-06T22:23:33.656Z 7 140-159 169 169 169 307 307 307 F F F
183902024-02-06T22:23:33.656Z 8 160-179 169 169 169 307 307 307 F F F
183912024-02-06T22:23:33.656Z 9 180-199 166 166 166 302 302 302 F F F
183922024-02-06T22:23:33.656Z 10 200-219 168 168 168 305 305 305 F F F
183932024-02-06T22:23:33.656Z 11 220-239 164 164 164 300 300 300 F F F
183942024-02-06T22:23:33.657Z 12 240-259 169 169 169 307 307 307 F F F
183952024-02-06T22:23:33.657Z 13 260-279 169 169 169 307 307 307 F F F
183962024-02-06T22:23:33.657Z 14 280-299 166 166 166 303 303 303 F F F
183972024-02-06T22:23:33.657Z 15 300-319 165 165 165 301 301 301 F F F
183982024-02-06T22:23:33.657Z 16 320-339 169 169 169 307 307 307 F F F
183992024-02-06T22:23:33.657Z 17 340-359 169 169 169 307 307 307 F F F
184002024-02-06T22:23:33.657Z 18 360-379 145 145 145 272 272 272 F F F
184012024-02-06T22:23:33.657Z 19 380-399 169 169 169 307 307 307 F F F
184022024-02-06T22:23:33.657Z 20 400-419 168 168 168 306 306 306 F F F
184032024-02-06T22:23:33.657Z 21 420-439 164 164 164 300 300 300 F F F
184042024-02-06T22:23:33.657Z 22 440-459 169 169 169 307 307 307 F F F
184052024-02-06T22:23:33.657Z 23 460-479 163 163 163 299 299 299 F F F
184062024-02-06T22:23:33.657Z 24 480-499 169 169 169 307 307 307 F F F
184072024-02-06T22:23:33.657Z 25 500-519 166 166 166 302 302 302 F F F
184082024-02-06T22:23:33.657Z 26 520-539 155 155 155 288 288 288 F F F
184092024-02-06T22:23:33.657Z 27 540-559 169 169 169 307 307 307 F F F
184102024-02-06T22:23:33.657Z 28 560-579 169 169 169 307 307 307 F F F
184112024-02-06T22:23:33.657Z 29 580-599 169 169 169 307 307 307 F F F
184122024-02-06T22:23:33.657ZMax gen: 169, Max flush: 307
184132024-02-06T22:23:33.657Z
184142024-02-06T22:23:33.657ZBegin loop 85
184152024-02-06T22:23:33.658ZDownstairs to restart: 1
184162024-02-06T22:23:33.674Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.849698681Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818}
184172024-02-06T22:23:33.679Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.85000072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818}
184182024-02-06T22:23:33.679ZCrucible runtime is spawned
184192024-02-06T22:23:33.679Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850060052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818}
184202024-02-06T22:23:33.679Z{"msg":"Crucible d6987e66-e05c-4294-bc51-4ddec3199a82 has session id: 41b019d2-b297-40a7-943c-b8dc41a7ca02","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850091998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184212024-02-06T22:23:33.679Z{"msg":"Upstairs opts: Upstairs UUID: d6987e66-e05c-4294-bc51-4ddec3199a82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850153929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184222024-02-06T22:23:33.679Z{"msg":"Crucible stats registered with UUID: d6987e66-e05c-4294-bc51-4ddec3199a82","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850182795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184232024-02-06T22:23:33.679Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850207832Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818}
184242024-02-06T22:23:33.679Z{"msg":"d6987e66-e05c-4294-bc51-4ddec3199a82 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850275713Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184252024-02-06T22:23:33.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850305819Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184262024-02-06T22:23:33.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850332185Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184272024-02-06T22:23:33.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850359532Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184282024-02-06T22:23:33.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850386018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184292024-02-06T22:23:33.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850432052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184302024-02-06T22:23:33.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850459148Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184312024-02-06T22:23:33.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850507621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184322024-02-06T22:23:33.679Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850637364Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184332024-02-06T22:23:33.679Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850696026Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184342024-02-06T22:23:33.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.85073987Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184352024-02-06T22:23:33.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.850780055Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184362024-02-06T22:23:33.680Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.85081837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184372024-02-06T22:23:33.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.851340119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184382024-02-06T22:23:33.680Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.851400051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184392024-02-06T22:23:33.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.851441375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184402024-02-06T22:23:33.680Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.851470731Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184412024-02-06T22:23:33.680Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852009329Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184422024-02-06T22:23:33.680Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852046214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184432024-02-06T22:23:33.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852238308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184442024-02-06T22:23:33.680Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852362011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184452024-02-06T22:23:33.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852639424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184462024-02-06T22:23:33.680Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852680128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184472024-02-06T22:23:33.680Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852696516Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184482024-02-06T22:23:33.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852858674Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184492024-02-06T22:23:33.680Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852906917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184502024-02-06T22:23:33.680Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852924465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184512024-02-06T22:23:33.680Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.852947082Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184522024-02-06T22:23:33.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853386623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184532024-02-06T22:23:33.684Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853430897Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184542024-02-06T22:23:33.684Z{"msg":"[0]R flush_numbers[0..12]: [300, 299, 305, 302, 307, 295, 307, 307, 307, 302, 305, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853447974Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184552024-02-06T22:23:33.684Z{"msg":"[0]R generation[0..12]: [164, 163, 168, 166, 169, 160, 169, 169, 169, 166, 168, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853471691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184562024-02-06T22:23:33.684Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853487339Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184572024-02-06T22:23:33.685Z{"msg":"[1]R flush_numbers[0..12]: [300, 299, 305, 302, 307, 295, 307, 307, 307, 302, 305, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853510106Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184582024-02-06T22:23:33.685Z{"msg":"[1]R generation[0..12]: [164, 163, 168, 166, 169, 160, 169, 169, 169, 166, 168, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853524804Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184592024-02-06T22:23:33.685Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853547041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184602024-02-06T22:23:33.685Z{"msg":"[2]R flush_numbers[0..12]: [300, 299, 305, 302, 307, 295, 307, 307, 307, 302, 305, 300]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853561469Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184612024-02-06T22:23:33.685Z{"msg":"[2]R generation[0..12]: [164, 163, 168, 166, 169, 160, 169, 169, 169, 166, 168, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853583076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184622024-02-06T22:23:33.685Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853597704Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
184632024-02-06T22:23:33.685Z,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184642024-02-06T22:23:33.685Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853622651Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184652024-02-06T22:23:33.685Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853636459Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184662024-02-06T22:23:33.685Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853657536Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184672024-02-06T22:23:33.685Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853671984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184682024-02-06T22:23:33.685Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853698261Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184692024-02-06T22:23:33.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853712988Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184702024-02-06T22:23:33.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853734805Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184712024-02-06T22:23:33.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853748584Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184722024-02-06T22:23:33.685Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853769221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184732024-02-06T22:23:33.685Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853782149Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184742024-02-06T22:23:33.685Z{"msg":"d6987e66-e05c-4294-bc51-4ddec3199a82 is now active with session: 41b019d2-b297-40a7-943c-b8dc41a7ca02","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853802946Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184752024-02-06T22:23:33.685Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853833822Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818,"session_id":"41b019d2-b297-40a7-943c-b8dc41a7ca02"}
184762024-02-06T22:23:33.685Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:32.853865918Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818}
184772024-02-06T22:23:33.685ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184782024-02-06T22:23:33.685ZLoading write count information from file "/tmp/test_repair_verify.data"
184792024-02-06T22:23:33.685ZRun Repair workload
184802024-02-06T22:23:33.685Z01/30 Write block 233 len 32768 data: 11 15 16 15 14 9 9 8
184812024-02-06T22:23:33.686Z02/30 Write block 355 len 16384 data: 19 18 19 18
184822024-02-06T22:23:33.686Z03/30 Read block 362 len 8192
184832024-02-06T22:23:33.686Z04/30 Read block 505 len 32768
184842024-02-06T22:23:33.686Z05/30 Read block 108 len 36864
184852024-02-06T22:23:33.689Z06/30 Write block 280 len 40960 data: 18 17 15 11 12 10 8 7 12 13
184862024-02-06T22:23:33.689Z07/30 Write block 68 len 28672 data: 13 12 11 11 12 12 15
184872024-02-06T22:23:33.689Z08/30 Write block 539 len 32768 data: 8 9 11 11 12 14 12 14
184882024-02-06T22:23:33.689Z09/30 Read block 376 len 8192
184892024-02-06T22:23:33.689Z10/30 Flush
184902024-02-06T22:23:33.692Z11/30 Write block 161 len 32768 data: 15 19 16 15 12 13 16 19
184912024-02-06T22:23:33.692Z12/30 Write block 143 len 24576 data: 15 15 16 18 19 20
184922024-02-06T22:23:33.692Z13/30 Read block 117 len 12288
184932024-02-06T22:23:33.695Z14/30 Flush
184942024-02-06T22:23:33.695Z15/30 Flush
184952024-02-06T22:23:33.695Z16/30 Write block 541 len 20480 data: 12 12 13 15 13
184962024-02-06T22:23:33.698Z17/30 Read block 295 len 4096
184972024-02-06T22:23:33.698Z18/30 Write block 450 len 32768 data: 10 11 11 13 18 17 12 11
184982024-02-06T22:23:33.698Z19/30 Read block 468 len 8192
184992024-02-06T22:23:33.701Z20/30 Read block 60 len 36864
185002024-02-06T22:23:33.701Z21/30 Read block 35 len 12288
185012024-02-06T22:23:33.701Z22/30 Write block 186 len 20480 data: 21 20 16 15 12
185022024-02-06T22:23:33.701Z23/30 Write block 232 len 20480 data: 13 12 16 17 16
185032024-02-06T22:23:33.705Z24/30 Read block 560 len 12288
185042024-02-06T22:23:33.705Z25/30 Read block 327 len 20480
185052024-02-06T22:23:33.705Z26/30 Read block 373 len 8192
185062024-02-06T22:23:33.705Z27/30 Write block 466 len 24576 data: 9 13 13 10 9 10
185072024-02-06T22:23:33.711Z28/30 Write block 299 len 8192 data: 11 14
185082024-02-06T22:23:33.711Z29/30 Write block 262 len 16384 data: 21 22 20 22
185092024-02-06T22:23:33.711Z30/30 Read block 160 len 4096
185102024-02-06T22:23:33.711Z----------------------------------------------------------------
185112024-02-06T22:23:33.712Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
185122024-02-06T22:23:33.712ZStates: Active Active Active
185132024-02-06T22:23:33.712ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185142024-02-06T22:23:33.712Z 1 Acked 1000 Write 8 Done Done Done false
185152024-02-06T22:23:33.712Z 2 Acked 1001 Write 4 Done Sent Done false
185162024-02-06T22:23:33.712Z 3 Acked 1002 Read 2 Done Sent Done false
185172024-02-06T22:23:33.712Z 4 Acked 1003 Read 8 Done Sent Done false
185182024-02-06T22:23:33.712Z 5 Acked 1004 Read 9 Done Sent Done false
185192024-02-06T22:23:33.712Z 6 Acked 1005 Write 10 Done Sent Done false
185202024-02-06T22:23:33.712Z 7 Acked 1006 Write 7 Done Sent Done false
185212024-02-06T22:23:33.712Z 8 Acked 1007 Write 8 Done Sent Done false
185222024-02-06T22:23:33.712Z 9 Acked 1008 Read 2 Done Sent Done false
185232024-02-06T22:23:33.712Z 10 Acked 1009 Flush 0 Done Sent Done false
185242024-02-06T22:23:33.712Z 11 Acked 1010 Write 8 Done Sent Done false
185252024-02-06T22:23:33.712Z 12 Acked 1011 Write 6 Done Sent Done false
185262024-02-06T22:23:33.712Z 13 Acked 1012 Read 3 Done Sent Done false
185272024-02-06T22:23:33.712Z 14 Acked 1013 Flush 0 Done Sent Done false
185282024-02-06T22:23:33.712Z 15 Acked 1014 Flush 0 Done Sent Done false
185292024-02-06T22:23:33.712Z 16 Acked 1015 Write 5 Done Sent Done false
185302024-02-06T22:23:33.712Z 17 Acked 1016 Read 1 Done Sent Done false
185312024-02-06T22:23:33.712Z 18 Acked 1017 Write 8 Done Sent Done false
185322024-02-06T22:23:33.712Z 19 Acked 1018 Read 2 Done Sent Done false
185332024-02-06T22:23:33.712Z 20 Acked 1019 Read 9 Done Sent Done false
185342024-02-06T22:23:33.712Z 21 Acked 1020 Read 3 Done Sent Done false
185352024-02-06T22:23:33.713Z 22 Acked 1021 Write 5 Done Sent Done false
185362024-02-06T22:23:33.713Z 23 Acked 1022 Write 5 Done Sent Done false
185372024-02-06T22:23:33.713Z 24 Acked 1023 Read 3 Done Sent Done false
185382024-02-06T22:23:33.713Z 25 Acked 1024 Read 5 Done Sent Done false
185392024-02-06T22:23:33.713Z 26 Acked 1025 Read 2 Done Sent Done false
185402024-02-06T22:23:33.713Z 27 Acked 1026 Write 6 Done Sent Done false
185412024-02-06T22:23:33.713Z 28 Acked 1027 Write 2 Done Sent Done false
185422024-02-06T22:23:33.713Z 29 Acked 1028 Write 4 Done Sent Sent false
185432024-02-06T22:23:33.713Z 30 Acked 1029 Read 1 Done Sent Sent false
185442024-02-06T22:23:33.713Z STATES DS:0 DS:1 DS:2 TOTAL
185452024-02-06T22:23:33.713Z New 0 0 0 0
185462024-02-06T22:23:33.713Z Sent 0 29 2 31
185472024-02-06T22:23:33.713Z Done 30 1 28 59
185482024-02-06T22:23:33.713Z Skipped 0 0 0 0
185492024-02-06T22:23:33.713Z Error 0 0 0 0
185502024-02-06T22:23:33.713ZLast Flush: 1014 0 1014
185512024-02-06T22:23:33.713ZDownstairs last five completed:
185522024-02-06T22:23:33.713ZUpstairs last five completed: 30 29 28 27 26
185532024-02-06T22:23:33.713Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:32.88256769Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1818}
185542024-02-06T22:23:33.713ZWrote out file "/tmp/test_repair_verify.data"
185552024-02-06T22:23:33.718Z
185562024-02-06T22:23:33.718ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185572024-02-06T22:23:33.733ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185582024-02-06T22:23:33.733Z 0 000-019 164 164 164 300 300 300 F F F
185592024-02-06T22:23:33.733Z 1 020-039 163 163 163 299 299 299 F F F
185602024-02-06T22:23:33.733Z 2 040-059 168 168 168 305 305 305 F F F
185612024-02-06T22:23:33.733Z 3 060-079 170 166 170 308 302 308 F F F <---
185622024-02-06T22:23:33.733Z 4 080-099 169 169 169 307 307 307 F F F
185632024-02-06T22:23:33.733Z 5 100-119 160 160 160 295 295 295 F F F
185642024-02-06T22:23:33.733Z 6 120-139 169 169 169 307 307 307 F F F
185652024-02-06T22:23:33.733Z 7 140-159 170 169 170 309 307 309 F F F <---
185662024-02-06T22:23:33.733Z 8 160-179 170 169 170 309 307 309 F F F <---
185672024-02-06T22:23:33.733Z 9 180-199 166 166 166 302 302 302 T F T <---
185682024-02-06T22:23:33.736Z 10 200-219 168 168 168 305 305 305 F F F
185692024-02-06T22:23:33.736Z 11 220-239 170 164 170 308 300 308 T T T <---
185702024-02-06T22:23:33.736Z 12 240-259 170 169 170 308 307 308 F T F <---
185712024-02-06T22:23:33.736Z 13 260-279 169 169 169 307 307 307 T F T <---
185722024-02-06T22:23:33.737Z 14 280-299 170 166 170 308 303 308 T F T <---
185732024-02-06T22:23:33.737Z 15 300-319 165 165 165 301 301 301 T F T <---
185742024-02-06T22:23:33.737Z 16 320-339 169 169 169 307 307 307 F F F
185752024-02-06T22:23:33.737Z 17 340-359 170 169 170 308 307 308 F F F <---
185762024-02-06T22:23:33.737Z 18 360-379 145 145 145 272 272 272 F F F
185772024-02-06T22:23:33.737Z 19 380-399 169 169 169 307 307 307 F F F
185782024-02-06T22:23:33.737Z 20 400-419 168 168 168 306 306 306 F F F
185792024-02-06T22:23:33.737Z 21 420-439 164 164 164 300 300 300 F F F
185802024-02-06T22:23:33.737Z 22 440-459 169 169 169 307 307 307 T F T <---
185812024-02-06T22:23:33.737Z 23 460-479 163 163 163 299 299 299 T F T <---
185822024-02-06T22:23:33.737Z 24 480-499 169 169 169 307 307 307 F F F
185832024-02-06T22:23:33.737Z 25 500-519 166 166 166 302 302 302 F F F
185842024-02-06T22:23:33.737Z 26 520-539 170 155 170 308 288 308 F F F <---
185852024-02-06T22:23:33.737Z 27 540-559 170 169 170 308 307 308 T F T <---
185862024-02-06T22:23:33.737Z 28 560-579 169 169 169 307 307 307 F F F
185872024-02-06T22:23:33.737Z 29 580-599 169 169 169 307 307 307 F F F
185882024-02-06T22:23:33.737ZMax gen: 170, Max flush: 309
185892024-02-06T22:23:33.737ZError: Difference in extent metadata found!
185902024-02-06T22:23:33.737ZA Difference in extent metadata is expected here
185912024-02-06T22:23:33.737ZOn loop 85
185922024-02-06T22:23:33.737Z
185932024-02-06T22:23:33.741ZVerifying data now
185942024-02-06T22:23:39.531ZNo change after verify
185952024-02-06T22:23:39.531ZLoop: 85 Downstairs dump after verify (and repair):
185962024-02-06T22:23:39.538ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185972024-02-06T22:23:39.538Z 0 000-019 164 164 164 300 300 300 F F F
185982024-02-06T22:23:39.538Z 1 020-039 163 163 163 299 299 299 F F F
185992024-02-06T22:23:39.538Z 2 040-059 168 168 168 305 305 305 F F F
186002024-02-06T22:23:39.538Z 3 060-079 170 170 170 308 308 308 F F F
186012024-02-06T22:23:39.538Z 4 080-099 169 169 169 307 307 307 F F F
186022024-02-06T22:23:39.538Z 5 100-119 160 160 160 295 295 295 F F F
186032024-02-06T22:23:39.538Z 6 120-139 169 169 169 307 307 307 F F F
186042024-02-06T22:23:39.538Z 7 140-159 170 170 170 309 309 309 F F F
186052024-02-06T22:23:39.538Z 8 160-179 170 170 170 309 309 309 F F F
186062024-02-06T22:23:39.538Z 9 180-199 171 171 171 310 310 310 F F F
186072024-02-06T22:23:39.538Z 10 200-219 168 168 168 305 305 305 F F F
186082024-02-06T22:23:39.539Z 11 220-239 171 171 171 310 310 310 F F F
186092024-02-06T22:23:39.539Z 12 240-259 170 170 170 308 308 308 F F F
186102024-02-06T22:23:39.539Z 13 260-279 171 171 171 310 310 310 F F F
186112024-02-06T22:23:39.539Z 14 280-299 171 171 171 310 310 310 F F F
186122024-02-06T22:23:39.539Z 15 300-319 171 171 171 310 310 310 F F F
186132024-02-06T22:23:39.539Z 16 320-339 169 169 169 307 307 307 F F F
186142024-02-06T22:23:39.539Z 17 340-359 170 170 170 308 308 308 F F F
186152024-02-06T22:23:39.539Z 18 360-379 145 145 145 272 272 272 F F F
186162024-02-06T22:23:39.539Z 19 380-399 169 169 169 307 307 307 F F F
186172024-02-06T22:23:39.539Z 20 400-419 168 168 168 306 306 306 F F F
186182024-02-06T22:23:39.539Z 21 420-439 164 164 164 300 300 300 F F F
186192024-02-06T22:23:39.539Z 22 440-459 171 171 171 310 310 310 F F F
186202024-02-06T22:23:39.539Z 23 460-479 171 171 171 310 310 310 F F F
186212024-02-06T22:23:39.539Z 24 480-499 169 169 169 307 307 307 F F F
186222024-02-06T22:23:39.539Z 25 500-519 166 166 166 302 302 302 F F F
186232024-02-06T22:23:39.539Z 26 520-539 170 170 170 308 308 308 F F F
186242024-02-06T22:23:39.539Z 27 540-559 171 171 171 310 310 310 F F F
186252024-02-06T22:23:39.539Z 28 560-579 169 169 169 307 307 307 F F F
186262024-02-06T22:23:39.539Z 29 580-599 169 169 169 307 307 307 F F F
186272024-02-06T22:23:39.539ZMax gen: 171, Max flush: 310
186282024-02-06T22:23:39.539Z
186292024-02-06T22:23:39.539ZBegin loop 86
186302024-02-06T22:23:39.539ZDownstairs to restart: 0
186312024-02-06T22:23:39.539Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639241882Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826}
186322024-02-06T22:23:39.540Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639568698Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826}
186332024-02-06T22:23:39.540Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
186342024-02-06T22:23:39.540Z,"time":"2024-02-06T22:23:38.639606263Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826}
186352024-02-06T22:23:39.540Z{"msg":"Crucible cd838dc1-7f92-49f4-a127-f6d0771a599f has session id: fe2b1161-88d9-49f2-98f8-5cb4d8b2a001","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639632619Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186362024-02-06T22:23:39.540Z{"msg":"Upstairs opts: Upstairs UUID: cd838dc1-7f92-49f4-a127-f6d0771a599f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639669585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186372024-02-06T22:23:39.540Z{"msg":"Crucible stats registered with UUID: cd838dc1-7f92-49f4-a127-f6d0771a599f","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.63970222Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186382024-02-06T22:23:39.540Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639729816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826}
186392024-02-06T22:23:39.540Z{"msg":"cd838dc1-7f92-49f4-a127-f6d0771a599f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639763802Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186402024-02-06T22:23:39.540Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639787478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186412024-02-06T22:23:39.540Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639818984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186422024-02-06T22:23:39.540Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639863698Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186432024-02-06T22:23:39.540Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639890424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186442024-02-06T22:23:39.540Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639930729Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186452024-02-06T22:23:39.540Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.639966614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186462024-02-06T22:23:39.540Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640002189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186472024-02-06T22:23:39.540Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640098427Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186482024-02-06T22:23:39.540Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640133692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186492024-02-06T22:23:39.540Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640162098Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186502024-02-06T22:23:39.540Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640203632Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186512024-02-06T22:23:39.540Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640232858Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186522024-02-06T22:23:39.540Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640690567Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186532024-02-06T22:23:39.540Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640728051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186542024-02-06T22:23:39.541Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640744169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186552024-02-06T22:23:39.541Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.640760997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186562024-02-06T22:23:39.541Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.641276377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186572024-02-06T22:23:39.541Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.641423867Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186582024-02-06T22:23:39.541Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.641783009Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186592024-02-06T22:23:39.541Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.641828363Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186602024-02-06T22:23:39.541Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.641853949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186612024-02-06T22:23:39.541Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.641934119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186622024-02-06T22:23:39.541Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.641978362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186632024-02-06T22:23:39.541Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.642015608Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186642024-02-06T22:23:39.541Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.642161978Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186652024-02-06T22:23:39.541Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.642199083Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186662024-02-06T22:23:39.541Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.642628874Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186672024-02-06T22:23:39.541Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643052937Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186682024-02-06T22:23:39.541Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643102861Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186692024-02-06T22:23:39.541Z{"msg":"[0]R flush_numbers[0..12]: [300, 299, 305, 308, 307, 295, 307, 309, 309, 310, 305, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.64318101Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186702024-02-06T22:23:39.541Z{"msg":"[0]R generation[0..12]: [164, 163, 168, 170, 169, 160, 169, 170, 170, 171, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643209656Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186712024-02-06T22:23:39.541Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643225184Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186722024-02-06T22:23:39.541Z{"msg":"[1]R flush_numbers[0..12]: [300, 299, 305, 308, 307, 295, 307, 309, 309, 310, 305, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643247211Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186732024-02-06T22:23:39.541Z{"msg":"[1]R generation[0..12]: [164, 163, 168, 170, 169, 160, 169, 170, 170, 171, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643261689Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186742024-02-06T22:23:39.541Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643283096Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186752024-02-06T22:23:39.541Z{"msg":"[2]R flush_numbers[0..12]: [300, 299, 305, 308, 307, 295, 307, 309, 309, 310, 305, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643297734Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186762024-02-06T22:23:39.541Z{"msg":"[2]R generation[0..12]: [164, 163, 168, 170, 169, 160, 169, 170, 170, 171, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643324181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186772024-02-06T22:23:39.541Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643345778Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186782024-02-06T22:23:39.541Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643368915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
186792024-02-06T22:23:39.542Zfe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186802024-02-06T22:23:39.542Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643386172Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186812024-02-06T22:23:39.542Z{"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.64340043Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186822024-02-06T22:23:39.542Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643414699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186832024-02-06T22:23:39.542Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643428326Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186842024-02-06T22:23:39.542Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643447404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186852024-02-06T22:23:39.542Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643484629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186862024-02-06T22:23:39.542Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643498827Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186872024-02-06T22:23:39.542Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643518984Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186882024-02-06T22:23:39.542Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643532592Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186892024-02-06T22:23:39.542Z{"msg":"cd838dc1-7f92-49f4-a127-f6d0771a599f is now active with session: fe2b1161-88d9-49f2-98f8-5cb4d8b2a001","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643555349Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186902024-02-06T22:23:39.542Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643576917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826,"session_id":"fe2b1161-88d9-49f2-98f8-5cb4d8b2a001"}
186912024-02-06T22:23:39.542Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:38.643599003Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826}
186922024-02-06T22:23:39.542ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186932024-02-06T22:23:39.542ZLoading write count information from file "/tmp/test_repair_verify.data"
186942024-02-06T22:23:39.542ZRun Repair workload
186952024-02-06T22:23:39.542Z01/30 Write block 137 len 20480 data: 10 12 11 9 9
186962024-02-06T22:23:39.547Z02/30 Read block 587 len 16384
186972024-02-06T22:23:39.547Z03/30 Write block 566 len 12288 data: 14 15 16
186982024-02-06T22:23:39.547Z04/30 Read block 566 len 4096
186992024-02-06T22:23:39.547Z05/30 Read block 165 len 20480
187002024-02-06T22:23:39.547Z06/30 Write block 312 len 24576 data: 13 16 16 18 17 16
187012024-02-06T22:23:39.547Z07/30 Write block 104 len 28672 data: 5 8 12 14 17 13 14
187022024-02-06T22:23:39.547Z08/30 Flush
187032024-02-06T22:23:39.547Z09/30 Write block 75 len 4096 data: 13
187042024-02-06T22:23:39.547Z10/30 Read block 534 len 8192
187052024-02-06T22:23:39.547Z11/30 Flush
187062024-02-06T22:23:39.547Z12/30 Write block 552 len 20480 data: 13 14 13 13 14
187072024-02-06T22:23:39.547Z13/30 Read block 378 len 4096
187082024-02-06T22:23:39.547Z14/30 Write block 147 len 24576 data: 20 21 23 19 15 15
187092024-02-06T22:23:39.547Z15/30 Read block 569 len 40960
187102024-02-06T22:23:39.547Z16/30 Write block 93 len 32768 data: 5 7 10 10 12 12 12 11
187112024-02-06T22:23:39.547Z17/30 Flush
187122024-02-06T22:23:39.547Z18/30 Write block 303 len 16384 data: 16 14 17 16
187132024-02-06T22:23:39.547Z19/30 Write block 177 len 24576 data: 12 13 17 17 19 18
187142024-02-06T22:23:39.547Z20/30 Read block 405 len 8192
187152024-02-06T22:23:39.547Z21/30 Read block 515 len 16384
187162024-02-06T22:23:39.547Z22/30 Write block 560 len 4096 data: 10
187172024-02-06T22:23:39.548Z23/30 Write block 477 len 32768 data: 12 12 12 10 7 9 10 13
187182024-02-06T22:23:39.548Z24/30 Write block 479 len 32768 data: 13 11 8 10 11 14 12 13
187192024-02-06T22:23:39.548Z25/30 Write block 380 len 28672 data: 13 14 12 12 12 13 15
187202024-02-06T22:23:39.548Z26/30 Flush
187212024-02-06T22:23:39.548Z27/30 Write block 254 len 12288 data: 13 11 7
187222024-02-06T22:23:39.548Z28/30 Write block 50 len 28672 data: 17 21 21 24 28 25 25
187232024-02-06T22:23:39.548Z29/30 Write block 510 len 16384 data: 14 13 11 15
187242024-02-06T22:23:39.548Z30/30 Write block 204 len 24576 data: 16 16 13 11 9 13
187252024-02-06T22:23:39.548Z----------------------------------------------------------------
187262024-02-06T22:23:39.548Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
187272024-02-06T22:23:39.548ZStates: Active Active Active
187282024-02-06T22:23:39.548ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187292024-02-06T22:23:39.548Z 1 Acked 1000 Write 5 Sent Done Done false
187302024-02-06T22:23:39.548Z 2 Acked 1001 Read 4 Sent Done Done false
187312024-02-06T22:23:39.548Z 3 Acked 1002 Write 3 Sent Done Done false
187322024-02-06T22:23:39.548Z 4 Acked 1003 Read 1 Sent Done Done false
187332024-02-06T22:23:39.548Z 5 Acked 1004 Read 5 Sent Done Done false
187342024-02-06T22:23:39.548Z 6 Acked 1005 Write 6 Sent Done Done false
187352024-02-06T22:23:39.548Z 7 Acked 1006 Write 7 Sent Done Done false
187362024-02-06T22:23:39.548Z 8 Acked 1007 Flush 0 Sent Done Done false
187372024-02-06T22:23:39.548Z 9 Acked 1008 Write 1 Sent Done Done false
187382024-02-06T22:23:39.548Z 10 Acked 1009 Read 2 Sent Done Done false
187392024-02-06T22:23:39.548Z 11 Acked 1010 Flush 0 Sent Done Done false
187402024-02-06T22:23:39.549Z 12 Acked 1011 Write 5 Sent Done Done false
187412024-02-06T22:23:39.549Z 13 Acked 1012 Read 1 Sent Done Done false
187422024-02-06T22:23:39.549Z 14 Acked 1013 Write 6 Sent Done Done false
187432024-02-06T22:23:39.549Z 15 Acked 1014 Read 10 Sent Done Done false
187442024-02-06T22:23:39.549Z 16 Acked 1015 Write 8 Sent Done Done false
187452024-02-06T22:23:39.549Z 17 Acked 1016 Flush 0 Sent Done Done false
187462024-02-06T22:23:39.549Z 18 Acked 1017 Write 4 Sent Done Done false
187472024-02-06T22:23:39.549Z 19 Acked 1018 Write 6 Sent Done Done false
187482024-02-06T22:23:39.549Z 20 Acked 1019 Read 2 Sent Done Done false
187492024-02-06T22:23:39.549Z 21 Acked 1020 Read 4 Sent Done Done false
187502024-02-06T22:23:39.549Z 22 Acked 1021 Write 1 Sent Done Done false
187512024-02-06T22:23:39.549Z 23 Acked 1022 Write 8 Sent Done Done false
187522024-02-06T22:23:39.549Z 24 Acked 1023 Write 8 Sent Done Done false
187532024-02-06T22:23:39.549Z 25 Acked 1024 Write 7 Sent Done Done false
187542024-02-06T22:23:39.549Z 26 Acked 1025 Flush 0 Sent Done Done false
187552024-02-06T22:23:39.549Z 27 Acked 1026 Write 3 Sent Done Done false
187562024-02-06T22:23:39.549Z 28 Acked 1027 Write 7 Sent Done Done false
187572024-02-06T22:23:39.549Z 29 Acked 1028 Write 4 Sent Done Done false
187582024-02-06T22:23:39.549Z 30 Acked 1029 Write 6 Sent Sent Sent false
187592024-02-06T22:23:39.549Z STATES DS:0 DS:1 DS:2 TOTAL
187602024-02-06T22:23:39.549Z New 0 0 0 0
187612024-02-06T22:23:39.549Z Sent 30 1 1 32
187622024-02-06T22:23:39.549Z Done 0 29 29 58
187632024-02-06T22:23:39.549Z Skipped 0 0 0 0
187642024-02-06T22:23:39.549Z Error 0 0 0 0
187652024-02-06T22:23:39.549ZLast Flush: 0 1025 1025
187662024-02-06T22:23:39.550ZDownstairs last five completed:
187672024-02-06T22:23:39.550ZUpstairs last five completed: 30 29 28 27 26
187682024-02-06T22:23:39.550Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:38.67471054Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1826}
187692024-02-06T22:23:39.550ZWrote out file "/tmp/test_repair_verify.data"
187702024-02-06T22:23:39.550Z
187712024-02-06T22:23:39.550ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187722024-02-06T22:23:39.550ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187732024-02-06T22:23:39.550Z 0 000-019 164 164 164 300 300 300 F F F
187742024-02-06T22:23:39.550Z 1 020-039 163 163 163 299 299 299 F F F
187752024-02-06T22:23:39.550Z 2 040-059 168 168 168 305 305 305 F T T <---
187762024-02-06T22:23:39.550Z 3 060-079 170 172 172 308 312 312 F F F <---
187772024-02-06T22:23:39.550Z 4 080-099 169 172 172 307 313 313 F F F <---
187782024-02-06T22:23:39.550Z 5 100-119 160 172 172 295 313 313 F F F <---
187792024-02-06T22:23:39.550Z 6 120-139 169 172 172 307 311 311 F F F <---
187802024-02-06T22:23:39.550Z 7 140-159 170 172 172 309 313 313 F F F <---
187812024-02-06T22:23:39.550Z 8 160-179 170 172 172 309 314 314 F F F <---
187822024-02-06T22:23:39.550Z 9 180-199 171 172 172 310 314 314 F F F <---
187832024-02-06T22:23:39.550Z 10 200-219 168 168 168 305 305 305 F T T <---
187842024-02-06T22:23:39.550Z 11 220-239 171 171 171 310 310 310 F F F
187852024-02-06T22:23:39.550Z 12 240-259 170 170 170 308 308 308 F T T <---
187862024-02-06T22:23:39.550Z 13 260-279 171 171 171 310 310 310 F F F
187872024-02-06T22:23:39.550Z 14 280-299 171 171 171 310 310 310 F F F
187882024-02-06T22:23:39.550Z 15 300-319 171 172 172 310 314 314 F F F <---
187892024-02-06T22:23:39.550Z 16 320-339 169 169 169 307 307 307 F F F
187902024-02-06T22:23:39.550Z 17 340-359 170 170 170 308 308 308 F F F
187912024-02-06T22:23:39.551Z 18 360-379 145 145 145 272 272 272 F F F
187922024-02-06T22:23:39.551Z 19 380-399 169 172 172 307 314 314 F F F <---
187932024-02-06T22:23:39.551Z 20 400-419 168 168 168 306 306 306 F F F
187942024-02-06T22:23:39.551Z 21 420-439 164 164 164 300 300 300 F F F
187952024-02-06T22:23:39.551Z 22 440-459 171 171 171 310 310 310 F F F
187962024-02-06T22:23:39.554Z 23 460-479 171 172 172 310 314 314 F F F <---
187972024-02-06T22:23:39.554Z 24 480-499 169 172 172 307 314 314 F F F <---
187982024-02-06T22:23:39.554Z 25 500-519 166 166 166 302 302 302 F T T <---
187992024-02-06T22:23:39.554Z 26 520-539 170 170 170 308 308 308 F F F
188002024-02-06T22:23:39.554Z 27 540-559 171 172 172 310 313 313 F F F <---
188012024-02-06T22:23:39.554Z 28 560-579 169 172 172 307 314 314 F F F <---
188022024-02-06T22:23:39.555Z 29 580-599 169 169 169 307 307 307 F F F
188032024-02-06T22:23:39.555ZMax gen: 172, Max flush: 314
188042024-02-06T22:23:39.555ZError: Difference in extent metadata found!
188052024-02-06T22:23:39.555ZA Difference in extent metadata is expected here
188062024-02-06T22:23:39.555ZOn loop 86
188072024-02-06T22:23:39.555Z
188082024-02-06T22:23:39.555ZVerifying data now
188092024-02-06T22:23:44.330ZNo change after verify
188102024-02-06T22:23:44.331ZLoop: 86 Downstairs dump after verify (and repair):
188112024-02-06T22:23:44.346ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188122024-02-06T22:23:44.346Z 0 000-019 164 164 164 300 300 300 F F F
188132024-02-06T22:23:44.346Z 1 020-039 163 163 163 299 299 299 F F F
188142024-02-06T22:23:44.346Z 2 040-059 173 173 173 315 315 315 F F F
188152024-02-06T22:23:44.346Z 3 060-079 172 172 172 312 312 312 F F F
188162024-02-06T22:23:44.346Z 4 080-099 172 172 172 313 313 313 F F F
188172024-02-06T22:23:44.346Z 5 100-119 172 172 172 313 313 313 F F F
188182024-02-06T22:23:44.346Z 6 120-139 172 172 172 311 311 311 F F F
188192024-02-06T22:23:44.346Z 7 140-159 172 172 172 313 313 313 F F F
188202024-02-06T22:23:44.346Z 8 160-179 172 172 172 314 314 314 F F F
188212024-02-06T22:23:44.346Z 9 180-199 172 172 172 314 314 314 F F F
188222024-02-06T22:23:44.346Z 10 200-219 173 173 173 315 315 315 F F F
188232024-02-06T22:23:44.346Z 11 220-239 171 171 171 310 310 310 F F F
188242024-02-06T22:23:44.346Z 12 240-259 173 173 173 315 315 315 F F F
188252024-02-06T22:23:44.346Z 13 260-279 171 171 171 310 310 310 F F F
188262024-02-06T22:23:44.347Z 14 280-299 171 171 171 310 310 310 F F F
188272024-02-06T22:23:44.347Z 15 300-319 172 172 172 314 314 314 F F F
188282024-02-06T22:23:44.350Z 16 320-339 169 169 169 307 307 307 F F F
188292024-02-06T22:23:44.350Z 17 340-359 170 170 170 308 308 308 F F F
188302024-02-06T22:23:44.350Z 18 360-379 145 145 145 272 272 272 F F F
188312024-02-06T22:23:44.350Z 19 380-399 172 172 172 314 314 314 F F F
188322024-02-06T22:23:44.350Z 20 400-419 168 168 168 306 306 306 F F F
188332024-02-06T22:23:44.350Z 21 420-439 164 164 164 300 300 300 F F F
188342024-02-06T22:23:44.350Z 22 440-459 171 171 171 310 310 310 F F F
188352024-02-06T22:23:44.350Z 23 460-479 172 172 172 314 314 314 F F F
188362024-02-06T22:23:44.350Z 24 480-499 172 172 172 314 314 314 F F F
188372024-02-06T22:23:44.350Z 25 500-519 173 173 173 315 315 315 F F F
188382024-02-06T22:23:44.350Z 26 520-539 170 170 170 308 308 308 F F F
188392024-02-06T22:23:44.350Z 27 540-559 172 172 172 313 313 313 F F F
188402024-02-06T22:23:44.350Z 28 560-579 172 172 172 314 314 314 F F F
188412024-02-06T22:23:44.350Z 29 580-599 169 169 169 307 307 307 F F F
188422024-02-06T22:23:44.351ZMax gen: 173, Max flush: 315
188432024-02-06T22:23:44.351Z
188442024-02-06T22:23:44.351ZBegin loop 87
188452024-02-06T22:23:44.351ZDownstairs to restart: 2
188462024-02-06T22:23:44.368Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.541806357Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834}
188472024-02-06T22:23:44.368Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542035166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834}
188482024-02-06T22:23:44.368Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542056483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834}
188492024-02-06T22:23:44.368Z{"msg":"Crucible 2216a204-d5c2-4c1b-ae96-a918b8be93e9 has session id: 731f8c0d-e76c-433c-9a9c-2a14fc126573","v":0,"name":"Crucible runtime is spawned
188502024-02-06T22:23:44.373Zcrucible","level":30,"time":"2024-02-06T22:23:43.542096878Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188512024-02-06T22:23:44.373Z{"msg":"Upstairs opts: Upstairs UUID: 2216a204-d5c2-4c1b-ae96-a918b8be93e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542146351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188522024-02-06T22:23:44.373Z{"msg":"Crucible stats registered with UUID: 2216a204-d5c2-4c1b-ae96-a918b8be93e9","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542174077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188532024-02-06T22:23:44.373Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542198264Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834}
188542024-02-06T22:23:44.373Z{"msg":"2216a204-d5c2-4c1b-ae96-a918b8be93e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542245158Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188552024-02-06T22:23:44.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542271994Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188562024-02-06T22:23:44.373Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542288002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188572024-02-06T22:23:44.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542308289Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188582024-02-06T22:23:44.373Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542331056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188592024-02-06T22:23:44.373Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542364372Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188602024-02-06T22:23:44.373Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542390548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188612024-02-06T22:23:44.373Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542427033Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188622024-02-06T22:23:44.373Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542461268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188632024-02-06T22:23:44.373Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542498643Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188642024-02-06T22:23:44.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.54252601Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188652024-02-06T22:23:44.373Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:43.542572333Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188662024-02-06T22:23:44.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.54260182Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188672024-02-06T22:23:44.373Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:43.542649943Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188682024-02-06T22:23:44.374Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:43.542679599Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188692024-02-06T22:23:44.374Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542724013Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188702024-02-06T22:23:44.374Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.54274564Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188712024-02-06T22:23:44.374Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.542827189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188722024-02-06T22:23:44.374Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.543301965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188732024-02-06T22:23:44.374Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.54333953Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188742024-02-06T22:23:44.374Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.543355138Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188752024-02-06T22:23:44.374Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.543370405Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188762024-02-06T22:23:44.374Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.543917152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188772024-02-06T22:23:44.374Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.543959976Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188782024-02-06T22:23:44.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.544466028Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188792024-02-06T22:23:44.374Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.544510631Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188802024-02-06T22:23:44.374Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.544529779Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188812024-02-06T22:23:44.374Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.544557605Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188822024-02-06T22:23:44.374Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.544572463Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188832024-02-06T22:23:44.374Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:43.54459432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188842024-02-06T22:23:45.371Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.544165446Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188852024-02-06T22:23:45.371Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.544377128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188862024-02-06T22:23:45.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.545197237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188872024-02-06T22:23:45.376Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.545233382Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188882024-02-06T22:23:45.376Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.545713377Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188892024-02-06T22:23:45.376Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546258743Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188902024-02-06T22:23:45.376Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546304607Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188912024-02-06T22:23:45.376Z{"msg":"[0]R flush_numbers[0..12]: [300, 299, 315, 312, 313, 313, 311, 313, 314, 314, 315, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546325074Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188922024-02-06T22:23:45.376Z{"msg":"[0]R generation[0..12]: [164, 163, 173, 172, 172, 172, 172, 172, 172, 172, 173, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546378977Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188932024-02-06T22:23:45.376Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546398174Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188942024-02-06T22:23:45.376ZWait for a query_work_queue command to finish before sending IO
188952024-02-06T22:23:45.376Z{"msg":"[1]R flush_numbers[0..12]: [300, 299, 315, 312, 313, 313, 311, 313, 314, 314, 315, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546439569Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188962024-02-06T22:23:45.376Z{"msg":"[1]R generation[0..12]: [164, 163, 173, 172, 172, 172, 172, 172, 172, 172, 173, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546480323Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188972024-02-06T22:23:45.376Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.54650061Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188982024-02-06T22:23:45.376Z{"msg":"[2]R flush_numbers[0..12]: [300, 299, 315, 312, 313, 313, 311, 313, 314, 314, 315, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546533716Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
188992024-02-06T22:23:45.376Z{"msg":"[2]R generation[0..12]: [164, 163, 173, 172, 172, 172, 172, 172, 172, 172, 173, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546588958Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189002024-02-06T22:23:45.376Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546615855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189012024-02-06T22:23:45.376Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546657239Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189022024-02-06T22:23:45.376Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546687595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189032024-02-06T22:23:45.376Z{"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546704993Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189042024-02-06T22:23:45.376Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546735459Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189052024-02-06T22:23:45.376Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546760445Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189062024-02-06T22:23:45.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546787412Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189072024-02-06T22:23:45.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546811939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189082024-02-06T22:23:45.377ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189092024-02-06T22:23:45.377Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546843764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189102024-02-06T22:23:45.377Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546866351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189112024-02-06T22:23:45.377Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546888708Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189122024-02-06T22:23:45.377Z{"msg":"2216a204-d5c2-4c1b-ae96-a918b8be93e9 is now active with session: 731f8c0d-e76c-433c-9a9c-2a14fc126573","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546902346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189132024-02-06T22:23:45.377Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546922113Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834,"session_id":"731f8c0d-e76c-433c-9a9c-2a14fc126573"}
189142024-02-06T22:23:45.377Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:44.546935332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834}
189152024-02-06T22:23:45.377ZLoading write count information from file "/tmp/test_repair_verify.data"
189162024-02-06T22:23:45.377ZRun Repair workload
189172024-02-06T22:23:45.377Z01/30 Write block 198 len 28672 data: 10 10 9 12 16 15 17
189182024-02-06T22:23:45.380Z02/30 Flush
189192024-02-06T22:23:45.380Z03/30 Write block 489 len 16384 data: 12 10 11 14
189202024-02-06T22:23:45.380Z04/30 Read block 146 len 40960
189212024-02-06T22:23:45.380Z05/30 Read block 11 len 36864
189222024-02-06T22:23:45.384Z06/30 Read block 407 len 8192
189232024-02-06T22:23:45.384Z07/30 Write block 17 len 8192 data: 11 12
189242024-02-06T22:23:45.384Z08/30 Write block 466 len 40960 data: 10 14 14 11 10 11 11 10 11 12
189252024-02-06T22:23:45.384Z09/30 Write block 8 len 40960 data: 8 7 11 9 10 9 12 13 12 12
189262024-02-06T22:23:45.384Z10/30 Read block 535 len 32768
189272024-02-06T22:23:45.387Z11/30 Write block 99 len 4096 data: 13
189282024-02-06T22:23:45.387Z12/30 Read block 499 len 28672
189292024-02-06T22:23:45.390Z13/30 Write block 249 len 8192 data: 17 14
189302024-02-06T22:23:45.390Z14/30 Read block 518 len 8192
189312024-02-06T22:23:45.390Z15/30 Read block 260 len 16384
189322024-02-06T22:23:45.390Z16/30 Read block 122 len 12288
189332024-02-06T22:23:45.396Z17/30 Write block 355 len 4096 data: 20
189342024-02-06T22:23:45.396Z18/30 Read block 14 len 28672
189352024-02-06T22:23:45.396Z19/30 Read block 54 len 8192
189362024-02-06T22:23:45.396Z20/30 Read block 190 len 40960
189372024-02-06T22:23:45.396Z21/30 Read block 303 len 40960
189382024-02-06T22:23:45.399Z22/30 Flush
189392024-02-06T22:23:45.399Z23/30 Write block 95 len 36864 data: 11 11 13 13 14 12 8 8 6
189402024-02-06T22:23:45.399Z24/30 Read block 311 len 24576
189412024-02-06T22:23:45.402Z25/30 Write block 311 len 4096 data: 14
189422024-02-06T22:23:45.402Z26/30 Write block 84 len 16384 data: 11 12 13 15
189432024-02-06T22:23:45.402Z27/30 Write block 498 len 40960 data: 16 15 10 10 11 15 13 16 17 16
189442024-02-06T22:23:45.402Z28/30 Read block 58 len 4096
189452024-02-06T22:23:45.406Z29/30 Read block 312 len 36864
189462024-02-06T22:23:45.406Z30/30 Read block 579 len 28672
189472024-02-06T22:23:45.406Z----------------------------------------------------------------
189482024-02-06T22:23:45.406Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:28
189492024-02-06T22:23:45.406ZStates: Active Active Active
189502024-02-06T22:23:45.406ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189512024-02-06T22:23:45.406Z 3 Acked 1002 Write 4 Done Done Done false
189522024-02-06T22:23:45.406Z 4 Acked 1003 Read 10 Done Done Done false
189532024-02-06T22:23:45.406Z 5 Acked 1004 Read 9 Done Done Done false
189542024-02-06T22:23:45.406Z 6 Acked 1005 Read 2 Done Done Done false
189552024-02-06T22:23:45.406Z 7 Acked 1006 Write 2 Done Done Sent false
189562024-02-06T22:23:45.406Z 8 Acked 1007 Write 10 Done Done Sent false
189572024-02-06T22:23:45.406Z 9 Acked 1008 Write 10 Done Done Sent false
189582024-02-06T22:23:45.407Z 10 Acked 1009 Read 8 Done Done Sent false
189592024-02-06T22:23:45.407Z 11 Acked 1010 Write 1 Done Done Sent false
189602024-02-06T22:23:45.407Z 12 Acked 1011 Read 7 Done Done Sent false
189612024-02-06T22:23:45.407Z 13 Acked 1012 Write 2 Done Done Sent false
189622024-02-06T22:23:45.407Z 14 Acked 1013 Read 2 Done Done Sent false
189632024-02-06T22:23:45.407Z 15 Acked 1014 Read 4 Done Done Sent false
189642024-02-06T22:23:45.407Z 16 Acked 1015 Read 3 Done Done Sent false
189652024-02-06T22:23:45.407Z 17 Acked 1016 Write 1 Done Done Sent false
189662024-02-06T22:23:45.407Z 18 Acked 1017 Read 7 Done Done Sent false
189672024-02-06T22:23:45.407Z 19 Acked 1018 Read 2 Done Done Sent false
189682024-02-06T22:23:45.407Z 20 Acked 1019 Read 10 Done Done Sent false
189692024-02-06T22:23:45.407Z 21 Acked 1020 Read 10 Done Done Sent false
189702024-02-06T22:23:45.407Z 22 Acked 1021 Flush 0 Done Done Sent false
189712024-02-06T22:23:45.407Z 23 Acked 1022 Write 9 Done Done Sent false
189722024-02-06T22:23:45.407Z 24 Acked 1023 Read 6 Done Done Sent false
189732024-02-06T22:23:45.407Z 25 Acked 1024 Write 1 Done Done Sent false
189742024-02-06T22:23:45.407Z 26 Acked 1025 Write 4 Done Done Sent false
189752024-02-06T22:23:45.407Z 27 Acked 1026 Write 10 Done Done Sent false
189762024-02-06T22:23:45.407Z 28 Acked 1027 Read 1 Done Done Sent false
189772024-02-06T22:23:45.407Z 29 Acked 1028 Read 9 Done Done Sent false
189782024-02-06T22:23:45.407Z 30 Acked 1029 Read 7 Done Done Sent false
189792024-02-06T22:23:45.407Z STATES DS:0 DS:1 DS:2 TOTAL
189802024-02-06T22:23:45.407Z New 0 0 0 0
189812024-02-06T22:23:45.407Z Sent 0 0 24 24
189822024-02-06T22:23:45.408Z Done 28 28 4 60
189832024-02-06T22:23:45.408Z Skipped 0 0 0 0
189842024-02-06T22:23:45.408Z Error 0 0 0 0
189852024-02-06T22:23:45.408ZLast Flush: 1021 1021 1001
189862024-02-06T22:23:45.408ZDownstairs last five completed: 1001 1000
189872024-02-06T22:23:45.408ZUpstairs last five completed: 30 29 28 27 26
189882024-02-06T22:23:45.408Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:44.578808824Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1834}
189892024-02-06T22:23:45.408ZWrote out file "/tmp/test_repair_verify.data"
189902024-02-06T22:23:45.411Z
189912024-02-06T22:23:45.414ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189922024-02-06T22:23:45.431ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189932024-02-06T22:23:45.431Z 0 000-019 174 174 164 317 317 300 F F F <---
189942024-02-06T22:23:45.431Z 1 020-039 163 163 163 299 299 299 F F F
189952024-02-06T22:23:45.431Z 2 040-059 173 173 173 315 315 315 F F F
189962024-02-06T22:23:45.432Z 3 060-079 172 172 172 312 312 312 F F F
189972024-02-06T22:23:45.432Z 4 080-099 174 174 172 317 317 313 T T F <---
189982024-02-06T22:23:45.432Z 5 100-119 172 172 172 313 313 313 T T F <---
189992024-02-06T22:23:45.432Z 6 120-139 172 172 172 311 311 311 F F F
190002024-02-06T22:23:45.432Z 7 140-159 172 172 172 313 313 313 F F F
190012024-02-06T22:23:45.432Z 8 160-179 172 172 172 314 314 314 F F F
190022024-02-06T22:23:45.432Z 9 180-199 174 174 174 316 316 316 F F F
190032024-02-06T22:23:45.432Z 10 200-219 174 174 174 316 316 316 F F F
190042024-02-06T22:23:45.432Z 11 220-239 171 171 171 310 310 310 F F F
190052024-02-06T22:23:45.432Z 12 240-259 174 174 173 317 317 315 F F F <---
190062024-02-06T22:23:45.435Z 13 260-279 171 171 171 310 310 310 F F F
190072024-02-06T22:23:45.435Z 14 280-299 171 171 171 310 310 310 F F F
190082024-02-06T22:23:45.435Z 15 300-319 172 172 172 314 314 314 T T F <---
190092024-02-06T22:23:45.435Z 16 320-339 169 169 169 307 307 307 F F F
190102024-02-06T22:23:45.435Z 17 340-359 174 174 170 317 317 308 F F F <---
190112024-02-06T22:23:45.435Z 18 360-379 145 145 145 272 272 272 F F F
190122024-02-06T22:23:45.436Z 19 380-399 172 172 172 314 314 314 F F F
190132024-02-06T22:23:45.436Z 20 400-419 168 168 168 306 306 306 F F F
190142024-02-06T22:23:45.436Z 21 420-439 164 164 164 300 300 300 F F F
190152024-02-06T22:23:45.436Z 22 440-459 171 171 171 310 310 310 F F F
190162024-02-06T22:23:45.436Z 23 460-479 174 174 172 317 317 314 F F F <---
190172024-02-06T22:23:45.436Z 24 480-499 174 174 172 317 317 314 T T T <---
190182024-02-06T22:23:45.436Z 25 500-519 173 173 173 315 315 315 T T F <---
190192024-02-06T22:23:45.436Z 26 520-539 170 170 170 308 308 308 F F F
190202024-02-06T22:23:45.436Z 27 540-559 172 172 172 313 313 313 F F F
190212024-02-06T22:23:45.436Z 28 560-579 172 172 172 314 314 314 F F F
190222024-02-06T22:23:45.436Z 29 580-599 169 169 169 307 307 307 F F F
190232024-02-06T22:23:45.436ZMax gen: 174, Max flush: 317
190242024-02-06T22:23:45.436ZError: Difference in extent metadata found!
190252024-02-06T22:23:45.436ZA Difference in extent metadata is expected here
190262024-02-06T22:23:45.436ZOn loop 87
190272024-02-06T22:23:45.436Z
190282024-02-06T22:23:45.439ZVerifying data now
190292024-02-06T22:23:49.914ZNo change after verify
190302024-02-06T22:23:49.914ZLoop: 87 Downstairs dump after verify (and repair):
190312024-02-06T22:23:49.929ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190322024-02-06T22:23:49.929Z 0 000-019 174 174 174 317 317 317 F F F
190332024-02-06T22:23:49.929Z 1 020-039 163 163 163 299 299 299 F F F
190342024-02-06T22:23:49.929Z 2 040-059 173 173 173 315 315 315 F F F
190352024-02-06T22:23:49.929Z 3 060-079 172 172 172 312 312 312 F F F
190362024-02-06T22:23:49.929Z 4 080-099 175 175 175 318 318 318 F F F
190372024-02-06T22:23:49.929Z 5 100-119 175 175 175 318 318 318 F F F
190382024-02-06T22:23:49.929Z 6 120-139 172 172 172 311 311 311 F F F
190392024-02-06T22:23:49.929Z 7 140-159 172 172 172 313 313 313 F F F
190402024-02-06T22:23:49.932Z 8 160-179 172 172 172 314 314 314 F F F
190412024-02-06T22:23:49.932Z 9 180-199 174 174 174 316 316 316 F F F
190422024-02-06T22:23:49.933Z 10 200-219 174 174 174 316 316 316 F F F
190432024-02-06T22:23:49.933Z 11 220-239 171 171 171 310 310 310 F F F
190442024-02-06T22:23:49.933Z 12 240-259 174 174 174 317 317 317 F F F
190452024-02-06T22:23:49.933Z 13 260-279 171 171 171 310 310 310 F F F
190462024-02-06T22:23:49.933Z 14 280-299 171 171 171 310 310 310 F F F
190472024-02-06T22:23:49.933Z 15 300-319 175 175 175 318 318 318 F F F
190482024-02-06T22:23:49.933Z 16 320-339 169 169 169 307 307 307 F F F
190492024-02-06T22:23:49.933Z 17 340-359 174 174 174 317 317 317 F F F
190502024-02-06T22:23:49.933Z 18 360-379 145 145 145 272 272 272 F F F
190512024-02-06T22:23:49.933Z 19 380-399 172 172 172 314 314 314 F F F
190522024-02-06T22:23:49.933Z 20 400-419 168 168 168 306 306 306 F F F
190532024-02-06T22:23:49.933Z 21 420-439 164 164 164 300 300 300 F F F
190542024-02-06T22:23:49.933Z 22 440-459 171 171 171 310 310 310 F F F
190552024-02-06T22:23:49.933Z 23 460-479 174 174 174 317 317 317 F F F
190562024-02-06T22:23:49.933Z 24 480-499 175 175 175 318 318 318 F F F
190572024-02-06T22:23:49.933Z 25 500-519 175 175 175 318 318 318 F F F
190582024-02-06T22:23:49.933Z 26 520-539 170 170 170 308 308 308 F F F
190592024-02-06T22:23:49.933Z 27 540-559 172 172 172 313 313 313 F F F
190602024-02-06T22:23:49.933Z 28 560-579 172 172 172 314 314 314 F F F
190612024-02-06T22:23:49.933Z 29 580-599 169 169 169 307 307 307 F F F
190622024-02-06T22:23:49.933ZMax gen: 175, Max flush: 318
190632024-02-06T22:23:49.934Z
190642024-02-06T22:23:49.934ZBegin loop 88
190652024-02-06T22:23:49.934ZDownstairs to restart: 2
190662024-02-06T22:23:49.949Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122091742Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842}
190672024-02-06T22:23:49.949Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"Crucible runtime is spawned
190682024-02-06T22:23:49.953Zcrucible","level":30,"time":"2024-02-06T22:23:49.122389642Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842}
190692024-02-06T22:23:49.954Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122436436Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842}
190702024-02-06T22:23:49.954Z{"msg":"Crucible b14b603f-e88a-48a0-b6b6-0ebae4be2371 has session id: 9bc102f5-6958-4627-91db-396b265944ea","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122468581Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190712024-02-06T22:23:49.954Z{"msg":"Upstairs opts: Upstairs UUID: b14b603f-e88a-48a0-b6b6-0ebae4be2371, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122531653Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190722024-02-06T22:23:49.954Z{"msg":"Crucible stats registered with UUID: b14b603f-e88a-48a0-b6b6-0ebae4be2371","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122572337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190732024-02-06T22:23:49.954Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122597444Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842}
190742024-02-06T22:23:49.954Z{"msg":"b14b603f-e88a-48a0-b6b6-0ebae4be2371 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.1226291Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190752024-02-06T22:23:49.954Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122664485Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190762024-02-06T22:23:49.954Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.12269891Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190772024-02-06T22:23:49.954Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122748454Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190782024-02-06T22:23:49.954Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122782319Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190792024-02-06T22:23:49.954Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122864158Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190802024-02-06T22:23:49.954Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122913331Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190812024-02-06T22:23:49.954Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122946907Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190822024-02-06T22:23:49.954Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.122983722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190832024-02-06T22:23:49.954Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.123052622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190842024-02-06T22:23:49.954Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.123089777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190852024-02-06T22:23:49.954Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.123131792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190862024-02-06T22:23:49.954Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.123153229Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190872024-02-06T22:23:49.954Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.123703394Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190882024-02-06T22:23:49.954Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.12374076Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190892024-02-06T22:23:49.954Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.123757177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190902024-02-06T22:23:49.954Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.123774255Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190912024-02-06T22:23:49.955Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.124367645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190922024-02-06T22:23:49.955Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.124408479Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190932024-02-06T22:23:49.955Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.124929449Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190942024-02-06T22:23:49.955Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.124972203Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190952024-02-06T22:23:49.955Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125088537Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190962024-02-06T22:23:49.955Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125122783Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190972024-02-06T22:23:49.955Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125137691Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190982024-02-06T22:23:49.955Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125174246Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
190992024-02-06T22:23:49.955Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125198023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191002024-02-06T22:23:49.955Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125230388Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191012024-02-06T22:23:49.955Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.12543835Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191022024-02-06T22:23:49.959Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125908336Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191032024-02-06T22:23:49.959Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125945542Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191042024-02-06T22:23:49.959Z{"msg":"[0]R flush_numbers[0..12]: [317, 299, 315, 312, 318, 318, 311, 313, 314, 316, 316, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.125963419Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191052024-02-06T22:23:49.959Z{"msg":"[0]R generation[0..12]: [174, 163, 173, 172, 175, 175, 172, 172, 172, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126012233Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191062024-02-06T22:23:49.959Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.12602796Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191072024-02-06T22:23:49.959Z{"msg":"[1]R flush_numbers[0..12]: [317, 299, 315, 312, 318, 318, 311, 313, 314, 316, 316, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126042609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191082024-02-06T22:23:49.959Z{"msg":"[1]R generation[0..12]: [174, 163, 173, 172, 175, 175, 172, 172, 172, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126064326Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191092024-02-06T22:23:49.959Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126080023Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191102024-02-06T22:23:49.960Z{"msg":"[2]R flush_numbers[0..12]: [317, 299, 315, 312, 318, 318, 311, 313, 314, 316, 316, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126094311Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191112024-02-06T22:23:49.960Z{"msg":"[2]R generation[0..12]: [174, 163, 173, 172, 175, 175, 172, 172, 172, 174, 174, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126114969Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191122024-02-06T22:23:49.960Z{"msg":"Wait for a query_work_queue command to finish before sending IO
191132024-02-06T22:23:49.960Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126130906Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191142024-02-06T22:23:49.960Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126145035Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191152024-02-06T22:23:49.960Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126165422Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191162024-02-06T22:23:49.960Z{"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.12617892Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191172024-02-06T22:23:49.960Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126192698Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191182024-02-06T22:23:49.960Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126212786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191192024-02-06T22:23:49.960Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126225484Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191202024-02-06T22:23:49.960Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126238922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191212024-02-06T22:23:49.960Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.1262522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191222024-02-06T22:23:49.960Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126265558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191232024-02-06T22:23:49.960Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126285406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191242024-02-06T22:23:49.960Z{"msg":"b14b603f-e88a-48a0-b6b6-0ebae4be2371 is now active with session: 9bc102f5-6958-4627-91db-396b265944ea","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126298934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191252024-02-06T22:23:49.960Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126311212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842,"session_id":"9bc102f5-6958-4627-91db-396b265944ea"}
191262024-02-06T22:23:49.960Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:49.126334379Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842}
191272024-02-06T22:23:49.960ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191282024-02-06T22:23:49.960ZLoading write count information from file "/tmp/test_repair_verify.data"
191292024-02-06T22:23:49.960ZRun Repair workload
191302024-02-06T22:23:49.960Z01/30 Write block 177 len 20480 data: 13 14 18 18 20
191312024-02-06T22:23:49.960Z02/30 Write block 368 len 24576 data: 13 12 12 10 8 11
191322024-02-06T22:23:49.960Z03/30 Write block 383 len 32768 data: 13 13 14 16 11 12 11 10
191332024-02-06T22:23:49.960Z04/30 Read block 97 len 40960
191342024-02-06T22:23:49.965Z05/30 Write block 212 len 20480 data: 12 15 19 16 13
191352024-02-06T22:23:49.966Z06/30 Read block 505 len 28672
191362024-02-06T22:23:49.966Z07/30 Read block 335 len 40960
191372024-02-06T22:23:49.966Z08/30 Write block 229 len 20480 data: 12 11 14 14 13
191382024-02-06T22:23:49.966Z09/30 Write block 510 len 20480 data: 15 14 12 16 17
191392024-02-06T22:23:49.966Z10/30 Read block 571 len 16384
191402024-02-06T22:23:49.966Z11/30 Write block 232 len 4096 data: 15
191412024-02-06T22:23:49.966Z12/30 Write block 529 len 20480 data: 15 17 18 16 19
191422024-02-06T22:23:49.969Z13/30 Read block 234 len 32768
191432024-02-06T22:23:49.969Z14/30 Read block 417 len 12288
191442024-02-06T22:23:49.969Z15/30 Read block 358 len 16384
191452024-02-06T22:23:49.969Z16/30 Flush
191462024-02-06T22:23:49.972Z17/30 Write block 209 len 8192 data: 14 13
191472024-02-06T22:23:49.972Z18/30 Read block 515 len 24576
191482024-02-06T22:23:49.972Z19/30 Flush
191492024-02-06T22:23:49.975Z20/30 Write block 176 len 16384 data: 13 14 15 19
191502024-02-06T22:23:49.975Z21/30 Flush
191512024-02-06T22:23:49.978Z22/30 Write block 443 len 28672 data: 9 9 10 10 8 10 10
191522024-02-06T22:23:49.978Z23/30 Write block 346 len 32768 data: 14 15 18 18 17 22 23 21
191532024-02-06T22:23:49.978Z24/30 Read block 296 len 12288
191542024-02-06T22:23:49.981Z25/30 Write block 481 len 40960 data: 9 11 12 15 13 14 13 12 13 11
191552024-02-06T22:23:49.981Z26/30 Read block 84 len 24576
191562024-02-06T22:23:49.981Z27/30 Read block 489 len 12288
191572024-02-06T22:23:49.985Z28/30 Write block 465 len 36864 data: 10 11 15 15 12 11 12 12 11
191582024-02-06T22:23:49.985Z29/30 Write block 501 len 40960 data: 11 12 16 14 17 18 17 15 15 16
191592024-02-06T22:23:49.985Z30/30 Write block 333 len 12288 data: 12 16 17
191602024-02-06T22:23:49.985Z----------------------------------------------------------------
191612024-02-06T22:23:49.985Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
191622024-02-06T22:23:49.985ZStates: Active Active Active
191632024-02-06T22:23:49.985ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191642024-02-06T22:23:49.985Z 1 Acked 1000 Write 5 Done Done Done false
191652024-02-06T22:23:49.985Z 2 Acked 1001 Write 6 Done Done Done false
191662024-02-06T22:23:49.985Z 3 Acked 1002 Write 8 Done Done Done false
191672024-02-06T22:23:49.985Z 4 Acked 1003 Read 10 Done Done Sent false
191682024-02-06T22:23:49.985Z 5 Acked 1004 Write 5 Done Done Sent false
191692024-02-06T22:23:49.985Z 6 Acked 1005 Read 7 Done Done Sent false
191702024-02-06T22:23:49.985Z 7 Acked 1006 Read 10 Done Done Sent false
191712024-02-06T22:23:49.985Z 8 Acked 1007 Write 5 Done Done Sent false
191722024-02-06T22:23:49.985Z 9 Acked 1008 Write 5 Done Done Sent false
191732024-02-06T22:23:49.985Z 10 Acked 1009 Read 4 Done Done Sent false
191742024-02-06T22:23:49.985Z 11 Acked 1010 Write 1 Done Done Sent false
191752024-02-06T22:23:49.985Z 12 Acked 1011 Write 5 Done Done Sent false
191762024-02-06T22:23:49.985Z 13 Acked 1012 Read 8 Done Done Sent false
191772024-02-06T22:23:49.986Z 14 Acked 1013 Read 3 Done Done Sent false
191782024-02-06T22:23:49.986Z 15 Acked 1014 Read 4 Done Done Sent false
191792024-02-06T22:23:49.986Z 16 Acked 1015 Flush 0 Done Done Sent false
191802024-02-06T22:23:49.986Z 17 Acked 1016 Write 2 Done Done Sent false
191812024-02-06T22:23:49.986Z 18 Acked 1017 Read 6 Done Done Sent false
191822024-02-06T22:23:49.986Z 19 Acked 1018 Flush 0 Done Done Sent false
191832024-02-06T22:23:49.986Z 20 Acked 1019 Write 4 Done Done Sent false
191842024-02-06T22:23:49.986Z 21 Acked 1020 Flush 0 Done Done Sent false
191852024-02-06T22:23:49.986Z 22 Acked 1021 Write 7 Done Done Sent false
191862024-02-06T22:23:49.986Z 23 Acked 1022 Write 8 Done Done Sent false
191872024-02-06T22:23:49.986Z 24 Acked 1023 Read 3 Done Done Sent false
191882024-02-06T22:23:49.986Z 25 Acked 1024 Write 10 Done Done Sent false
191892024-02-06T22:23:49.986Z 26 Acked 1025 Read 6 Done Done Sent false
191902024-02-06T22:23:49.986Z 27 Acked 1026 Read 3 Done Done Sent false
191912024-02-06T22:23:49.986Z 28 Acked 1027 Write 9 Done Done Sent false
191922024-02-06T22:23:49.986Z 29 Acked 1028 Write 10 Sent Sent Sent false
191932024-02-06T22:23:49.986Z 30 Acked 1029 Write 3 Sent Sent Sent false
191942024-02-06T22:23:49.986Z STATES DS:0 DS:1 DS:2 TOTAL
191952024-02-06T22:23:49.986Z New 0 0 0 0
191962024-02-06T22:23:49.986Z Sent 2 2 27 31
191972024-02-06T22:23:49.986Z Done 28 28 3 59
191982024-02-06T22:23:49.986Z Skipped 0 0 0 0
191992024-02-06T22:23:49.986Z Error 0 0 0 0
192002024-02-06T22:23:49.986ZLast Flush: 1020 1020 0
192012024-02-06T22:23:49.987ZDownstairs last five completed:
192022024-02-06T22:23:49.987ZUpstairs last five completed: 30 29 28 27 26
192032024-02-06T22:23:49.987Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:49.157388113Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1842}
192042024-02-06T22:23:49.987ZWrote out file "/tmp/test_repair_verify.data"
192052024-02-06T22:23:49.990Z
192062024-02-06T22:23:49.993ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192072024-02-06T22:23:50.009ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192082024-02-06T22:23:50.009Z 0 000-019 174 174 174 317 317 317 F F F
192092024-02-06T22:23:50.009Z 1 020-039 163 163 163 299 299 299 F F F
192102024-02-06T22:23:50.009Z 2 040-059 173 173 173 315 315 315 F F F
192112024-02-06T22:23:50.010Z 3 060-079 172 172 172 312 312 312 F F F
192122024-02-06T22:23:50.010Z 4 080-099 175 175 175 318 318 318 F F F
192132024-02-06T22:23:50.010Z 5 100-119 175 175 175 318 318 318 F F F
192142024-02-06T22:23:50.010Z 6 120-139 172 172 172 311 311 311 F F F
192152024-02-06T22:23:50.010Z 7 140-159 172 172 172 313 313 313 F F F
192162024-02-06T22:23:50.010Z 8 160-179 176 176 172 321 321 314 F F T <---
192172024-02-06T22:23:50.013Z 9 180-199 176 176 174 319 319 316 F F T <---
192182024-02-06T22:23:50.013Z 10 200-219 176 176 174 320 320 316 F F F <---
192192024-02-06T22:23:50.013Z 11 220-239 176 176 171 319 319 310 F F F <---
192202024-02-06T22:23:50.013Z 12 240-259 174 174 174 317 317 317 F F F
192212024-02-06T22:23:50.013Z 13 260-279 171 171 171 310 310 310 F F F
192222024-02-06T22:23:50.013Z 14 280-299 171 171 171 310 310 310 F F F
192232024-02-06T22:23:50.013Z 15 300-319 175 175 175 318 318 318 F F F
192242024-02-06T22:23:50.013Z 16 320-339 169 169 169 307 307 307 T T F <---
192252024-02-06T22:23:50.013Z 17 340-359 174 174 174 317 317 317 T T F <---
192262024-02-06T22:23:50.013Z 18 360-379 176 176 145 319 319 272 F F T <---
192272024-02-06T22:23:50.013Z 19 380-399 176 176 172 319 319 314 F F T <---
192282024-02-06T22:23:50.013Z 20 400-419 168 168 168 306 306 306 F F F
192292024-02-06T22:23:50.013Z 21 420-439 164 164 164 300 300 300 F F F
192302024-02-06T22:23:50.013Z 22 440-459 171 171 171 310 310 310 T T F <---
192312024-02-06T22:23:50.014Z 23 460-479 174 174 174 317 317 317 T T F <---
192322024-02-06T22:23:50.014Z 24 480-499 175 175 175 318 318 318 T T F <---
192332024-02-06T22:23:50.014Z 25 500-519 176 176 175 319 319 318 T T F <---
192342024-02-06T22:23:50.014Z 26 520-539 176 176 170 319 319 308 F F F <---
192352024-02-06T22:23:50.014Z 27 540-559 172 172 172 313 313 313 F F F
192362024-02-06T22:23:50.014Z 28 560-579 172 172 172 314 314 314 F F F
192372024-02-06T22:23:50.014Z 29 580-599 169 169 169 307 307 307 F F F
192382024-02-06T22:23:50.014ZMax gen: 176, Max flush: 321
192392024-02-06T22:23:50.014ZError: Difference in extent metadata found!
192402024-02-06T22:23:50.014ZA Difference in extent metadata is expected here
192412024-02-06T22:23:50.014ZOn loop 88
192422024-02-06T22:23:50.014Z
192432024-02-06T22:23:50.017ZVerifying data now
192442024-02-06T22:23:55.657ZNo change after verify
192452024-02-06T22:23:55.657ZLoop: 88 Downstairs dump after verify (and repair):
192462024-02-06T22:23:55.672ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192472024-02-06T22:23:55.673Z 0 000-019 174 174 174 317 317 317 F F F
192482024-02-06T22:23:55.673Z 1 020-039 163 163 163 299 299 299 F F F
192492024-02-06T22:23:55.673Z 2 040-059 173 173 173 315 315 315 F F F
192502024-02-06T22:23:55.673Z 3 060-079 172 172 172 312 312 312 F F F
192512024-02-06T22:23:55.676Z 4 080-099 175 175 175 318 318 318 F F F
192522024-02-06T22:23:55.676Z 5 100-119 175 175 175 318 318 318 F F F
192532024-02-06T22:23:55.676Z 6 120-139 172 172 172 311 311 311 F F F
192542024-02-06T22:23:55.676Z 7 140-159 172 172 172 313 313 313 F F F
192552024-02-06T22:23:55.676Z 8 160-179 176 176 176 321 321 321 F F F
192562024-02-06T22:23:55.676Z 9 180-199 176 176 176 319 319 319 F F F
192572024-02-06T22:23:55.676Z 10 200-219 176 176 176 320 320 320 F F F
192582024-02-06T22:23:55.676Z 11 220-239 176 176 176 319 319 319 F F F
192592024-02-06T22:23:55.676Z 12 240-259 174 174 174 317 317 317 F F F
192602024-02-06T22:23:55.677Z 13 260-279 171 171 171 310 310 310 F F F
192612024-02-06T22:23:55.677Z 14 280-299 171 171 171 310 310 310 F F F
192622024-02-06T22:23:55.677Z 15 300-319 175 175 175 318 318 318 F F F
192632024-02-06T22:23:55.677Z 16 320-339 177 177 177 322 322 322 F F F
192642024-02-06T22:23:55.677Z 17 340-359 177 177 177 322 322 322 F F F
192652024-02-06T22:23:55.677Z 18 360-379 176 176 176 319 319 319 F F F
192662024-02-06T22:23:55.677Z 19 380-399 176 176 176 319 319 319 F F F
192672024-02-06T22:23:55.677Z 20 400-419 168 168 168 306 306 306 F F F
192682024-02-06T22:23:55.677Z 21 420-439 164 164 164 300 300 300 F F F
192692024-02-06T22:23:55.677Z 22 440-459 177 177 177 322 322 322 F F F
192702024-02-06T22:23:55.677Z 23 460-479 177 177 177 322 322 322 F F F
192712024-02-06T22:23:55.677Z 24 480-499 177 177 177 322 322 322 F F F
192722024-02-06T22:23:55.677Z 25 500-519 177 177 177 322 322 322 F F F
192732024-02-06T22:23:55.677Z 26 520-539 176 176 176 319 319 319 F F F
192742024-02-06T22:23:55.677Z 27 540-559 172 172 172 313 313 313 F F F
192752024-02-06T22:23:55.677Z 28 560-579 172 172 172 314 314 314 F F F
192762024-02-06T22:23:55.677Z 29 580-599 169 169 169 307 307 307 F F F
192772024-02-06T22:23:55.677ZMax gen: 177, Max flush: 322
192782024-02-06T22:23:55.677Z
192792024-02-06T22:23:55.677ZBegin loop 89
192802024-02-06T22:23:55.677ZDownstairs to restart: 2
192812024-02-06T22:23:55.694Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.866335184Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850}
192822024-02-06T22:23:55.694Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
192832024-02-06T22:23:55.694Z,"time":"2024-02-06T22:23:54.866686857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850}
192842024-02-06T22:23:55.699Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.866720282Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850}
192852024-02-06T22:23:55.699Z{"msg":"Crucible c4b79088-0914-4493-99ad-0b3e5ac94607 has session id: 861ffbcf-eddc-4e1c-b308-de251f58f932","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.866767846Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192862024-02-06T22:23:55.699Z{"msg":"Upstairs opts: Upstairs UUID: c4b79088-0914-4493-99ad-0b3e5ac94607, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.866816709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192872024-02-06T22:23:55.699Z{"msg":"Crucible stats registered with UUID: c4b79088-0914-4493-99ad-0b3e5ac94607","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.866854854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192882024-02-06T22:23:55.699Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.866875721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850}
192892024-02-06T22:23:55.699Z{"msg":"c4b79088-0914-4493-99ad-0b3e5ac94607 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.866913676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192902024-02-06T22:23:55.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.86695787Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192912024-02-06T22:23:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867016292Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192922024-02-06T22:23:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867058256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192932024-02-06T22:23:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867100001Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192942024-02-06T22:23:55.700Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867131466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192952024-02-06T22:23:55.700Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867161062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192962024-02-06T22:23:55.700Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867212846Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192972024-02-06T22:23:55.700Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867301314Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192982024-02-06T22:23:55.700Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867349187Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
192992024-02-06T22:23:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867379053Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193002024-02-06T22:23:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867417808Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193012024-02-06T22:23:55.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.867455823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193022024-02-06T22:23:55.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.868029375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193032024-02-06T22:23:55.700Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.868072889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193042024-02-06T22:23:55.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.868098296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193052024-02-06T22:23:55.700Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.86813881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193062024-02-06T22:23:55.700Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.868738839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193072024-02-06T22:23:55.700Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.868792452Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193082024-02-06T22:23:55.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869241851Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193092024-02-06T22:23:55.700Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869295344Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193102024-02-06T22:23:55.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869486998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193112024-02-06T22:23:55.700Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869538461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193122024-02-06T22:23:55.701Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869610731Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193132024-02-06T22:23:55.701Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869627949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193142024-02-06T22:23:55.701Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869643677Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193152024-02-06T22:23:55.701Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869657955Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193162024-02-06T22:23:55.701Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.869839771Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193172024-02-06T22:23:55.705Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870319506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193182024-02-06T22:23:55.705Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.87036025Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193192024-02-06T22:23:55.705Z{"msg":"[0]R flush_numbers[0..12]: [317, 299, 315, 312, 318, 318, 311, 313, 321, 319, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870379188Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193202024-02-06T22:23:55.705Z{"msg":"[0]R generation[0..12]: [174, 163, 173, 172, 175, 175, 172, 172, 176, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870403594Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193212024-02-06T22:23:55.705Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870419212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193222024-02-06T22:23:55.705Z{"msg":"[1]R flush_numbers[0..12]: [317, 299, 315, 312, 318, 318, 311, 313, 321, 319, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.87043384Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193232024-02-06T22:23:55.705Z{"msg":"[1]R generation[0..12]: [174, 163, 173, 172, 175, 175, 172, 172, 176, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870461627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193242024-02-06T22:23:55.705Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870477494Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193252024-02-06T22:23:55.705Z{"msg":"[2]R flush_numbers[0..12]: [317, 299, 315, 312, 318, 318, 311, 313, 321, 319, 320, 319]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870491932Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193262024-02-06T22:23:55.705Z{"msg":"[2]R generation[0..12]: [174, 163, 173, 172, 175, 175, 172, 172, 176, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870517989Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193272024-02-06T22:23:55.706Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
193282024-02-06T22:23:55.706Z,"level":30,"time":"2024-02-06T22:23:54.870540336Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193292024-02-06T22:23:55.706Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870555464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193302024-02-06T22:23:55.706Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870569092Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193312024-02-06T22:23:55.706Z{"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870582Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193322024-02-06T22:23:55.706Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870614126Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193332024-02-06T22:23:55.706Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870635143Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193342024-02-06T22:23:55.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870660909Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193352024-02-06T22:23:55.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870688376Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193362024-02-06T22:23:55.706Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870703894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193372024-02-06T22:23:55.706Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870718032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193382024-02-06T22:23:55.706Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.87073047Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193392024-02-06T22:23:55.706Z{"msg":"c4b79088-0914-4493-99ad-0b3e5ac94607 is now active with session: 861ffbcf-eddc-4e1c-b308-de251f58f932","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870751967Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193402024-02-06T22:23:55.706Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870765765Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850,"session_id":"861ffbcf-eddc-4e1c-b308-de251f58f932"}
193412024-02-06T22:23:55.706Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:54.870778664Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850}
193422024-02-06T22:23:55.706ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193432024-02-06T22:23:55.706ZLoading write count information from file "/tmp/test_repair_verify.data"
193442024-02-06T22:23:55.706ZRun Repair workload
193452024-02-06T22:23:55.706Z01/30 Write block 462 len 36864 data: 12 10 11 11 12 16 16 13 12
193462024-02-06T22:23:55.706Z02/30 Write block 224 len 36864 data: 19 16 17 12 12 13 12 15 16
193472024-02-06T22:23:55.706Z03/30 Read block 203 len 24576
193482024-02-06T22:23:55.706Z04/30 Read block 534 len 24576
193492024-02-06T22:23:55.706Z05/30 Flush
193502024-02-06T22:23:55.710Z06/30 Flush
193512024-02-06T22:23:55.710Z07/30 Write block 491 len 8192 data: 12 15
193522024-02-06T22:23:55.710Z08/30 Read block 124 len 32768
193532024-02-06T22:23:55.710Z09/30 Write block 173 len 20480 data: 17 16 15 14 15
193542024-02-06T22:23:55.710Z10/30 Write block 315 len 12288 data: 19 18 17
193552024-02-06T22:23:55.710Z11/30 Read block 115 len 16384
193562024-02-06T22:23:55.713Z12/30 Read block 57 len 12288
193572024-02-06T22:23:55.713Z13/30 Read block 501 len 4096
193582024-02-06T22:23:55.714Z14/30 Read block 459 len 32768
193592024-02-06T22:23:55.714Z15/30 Write block 121 len 32768 data: 14 16 18 16 17 14 17 18
193602024-02-06T22:23:55.714Z16/30 Read block 190 len 8192
193612024-02-06T22:23:55.717Z17/30 Read block 92 len 28672
193622024-02-06T22:23:55.717Z18/30 Read block 69 len 4096
193632024-02-06T22:23:55.717Z19/30 Read block 276 len 4096
193642024-02-06T22:23:55.717Z20/30 Write block 326 len 8192 data: 8 8
193652024-02-06T22:23:55.723Z21/30 Read block 17 len 8192
193662024-02-06T22:23:55.723Z22/30 Write block 567 len 8192 data: 16 17
193672024-02-06T22:23:55.723Z23/30 Read block 31 len 8192
193682024-02-06T22:23:55.723Z24/30 Write block 574 len 8192 data: 19 15
193692024-02-06T22:23:55.723Z25/30 Read block 405 len 40960
193702024-02-06T22:23:55.727Z26/30 Write block 159 len 12288 data: 17 15 16
193712024-02-06T22:23:55.727Z27/30 Write block 28 len 8192 data: 15 13
193722024-02-06T22:23:55.727Z28/30 Write block 548 len 32768 data: 11 11 10 12 14 15 14 14
193732024-02-06T22:23:55.727Z29/30 Read block 103 len 12288
193742024-02-06T22:23:55.727Z30/30 Read block 598 len 8192
193752024-02-06T22:23:55.727Z----------------------------------------------------------------
193762024-02-06T22:23:55.727Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
193772024-02-06T22:23:55.727ZStates: Active Active Active
193782024-02-06T22:23:55.727ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193792024-02-06T22:23:55.727Z 1 Acked 1000 Write 9 Done Done Done false
193802024-02-06T22:23:55.727Z 2 Acked 1001 Write 9 Done Done Done false
193812024-02-06T22:23:55.727Z 3 Acked 1002 Read 6 Done Done Done false
193822024-02-06T22:23:55.727Z 4 Acked 1003 Read 6 Done Done Sent false
193832024-02-06T22:23:55.727Z 5 Acked 1004 Flush 0 Done Done Sent false
193842024-02-06T22:23:55.728Z 6 Acked 1005 Flush 0 Done Done Sent false
193852024-02-06T22:23:55.728Z 7 Acked 1006 Write 2 Done Done Sent false
193862024-02-06T22:23:55.728Z 8 Acked 1007 Read 8 Done Done Sent false
193872024-02-06T22:23:55.728Z 9 Acked 1008 Write 5 Done Done Sent false
193882024-02-06T22:23:55.728Z 10 Acked 1009 Write 3 Done Done Sent false
193892024-02-06T22:23:55.728Z 11 Acked 1010 Read 4 Done Done Sent false
193902024-02-06T22:23:55.728Z 12 Acked 1011 Read 3 Done Done Sent false
193912024-02-06T22:23:55.728Z 13 Acked 1012 Read 1 Done Done Sent false
193922024-02-06T22:23:55.728Z 14 Acked 1013 Read 8 Done Done Sent false
193932024-02-06T22:23:55.728Z 15 Acked 1014 Write 8 Done Done Sent false
193942024-02-06T22:23:55.728Z 16 Acked 1015 Read 2 Done Done Sent false
193952024-02-06T22:23:55.728Z 17 Acked 1016 Read 7 Done Done Sent false
193962024-02-06T22:23:55.728Z 18 Acked 1017 Read 1 Done Done Sent false
193972024-02-06T22:23:55.728Z 19 Acked 1018 Read 1 Done Done Sent false
193982024-02-06T22:23:55.728Z 20 Acked 1019 Write 2 Done Done Sent false
193992024-02-06T22:23:55.728Z 21 Acked 1020 Read 2 Done Done Sent false
194002024-02-06T22:23:55.728Z 22 Acked 1021 Write 2 Done Done Sent false
194012024-02-06T22:23:55.728Z 23 Acked 1022 Read 2 Done Done Sent false
194022024-02-06T22:23:55.728Z 24 Acked 1023 Write 2 Done Done Sent false
194032024-02-06T22:23:55.728Z 25 Acked 1024 Read 10 Done Done Sent false
194042024-02-06T22:23:55.728Z 26 Acked 1025 Write 3 Done Done Sent false
194052024-02-06T22:23:55.728Z 27 Acked 1026 Write 2 Done Done Sent false
194062024-02-06T22:23:55.728Z 28 Acked 1027 Write 8 Done Done Sent false
194072024-02-06T22:23:55.728Z 29 Acked 1028 Read 3 Done Done Sent false
194082024-02-06T22:23:55.729Z 30 Acked 1029 Read 2 Sent Done Sent false
194092024-02-06T22:23:55.729Z STATES DS:0 DS:1 DS:2 TOTAL
194102024-02-06T22:23:55.729Z New 0 0 0 0
194112024-02-06T22:23:55.729Z Sent 1 0 27 28
194122024-02-06T22:23:55.729Z Done 29 30 3 62
194132024-02-06T22:23:55.729Z Skipped 0 0 0 0
194142024-02-06T22:23:55.729Z Error 0 0 0 0
194152024-02-06T22:23:55.729ZLast Flush: 1005 1005 0
194162024-02-06T22:23:55.729ZDownstairs last five completed:
194172024-02-06T22:23:55.729ZUpstairs last five completed: 30 29 28 27 26
194182024-02-06T22:23:55.729Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:54.89678337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1850}
194192024-02-06T22:23:55.729ZWrote out file "/tmp/test_repair_verify.data"
194202024-02-06T22:23:55.732Z
194212024-02-06T22:23:55.732ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194222024-02-06T22:23:55.750ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194232024-02-06T22:23:55.750Z 0 000-019 174 174 174 317 317 317 F F F
194242024-02-06T22:23:55.750Z 1 020-039 163 163 163 299 299 299 T T F <---
194252024-02-06T22:23:55.750Z 2 040-059 173 173 173 315 315 315 F F F
194262024-02-06T22:23:55.750Z 3 060-079 172 172 172 312 312 312 F F F
194272024-02-06T22:23:55.750Z 4 080-099 175 175 175 318 318 318 F F F
194282024-02-06T22:23:55.750Z 5 100-119 175 175 175 318 318 318 F F F
194292024-02-06T22:23:55.750Z 6 120-139 172 172 172 311 311 311 T T F <---
194302024-02-06T22:23:55.750Z 7 140-159 172 172 172 313 313 313 T T F <---
194312024-02-06T22:23:55.750Z 8 160-179 176 176 176 321 321 321 T T F <---
194322024-02-06T22:23:55.751Z 9 180-199 176 176 176 319 319 319 F F F
194332024-02-06T22:23:55.751Z 10 200-219 176 176 176 320 320 320 F F F
194342024-02-06T22:23:55.751Z 11 220-239 178 178 176 323 323 319 F F T <---
194352024-02-06T22:23:55.754Z 12 240-259 174 174 174 317 317 317 F F F
194362024-02-06T22:23:55.754Z 13 260-279 171 171 171 310 310 310 F F F
194372024-02-06T22:23:55.754Z 14 280-299 171 171 171 310 310 310 F F F
194382024-02-06T22:23:55.754Z 15 300-319 175 175 175 318 318 318 T T F <---
194392024-02-06T22:23:55.754Z 16 320-339 177 177 177 322 322 322 T T F <---
194402024-02-06T22:23:55.754Z 17 340-359 177 177 177 322 322 322 F F F
194412024-02-06T22:23:55.754Z 18 360-379 176 176 176 319 319 319 F F F
194422024-02-06T22:23:55.754Z 19 380-399 176 176 176 319 319 319 F F F
194432024-02-06T22:23:55.754Z 20 400-419 168 168 168 306 306 306 F F F
194442024-02-06T22:23:55.754Z 21 420-439 164 164 164 300 300 300 F F F
194452024-02-06T22:23:55.754Z 22 440-459 177 177 177 322 322 322 F F F
194462024-02-06T22:23:55.754Z 23 460-479 178 178 177 323 323 322 F F T <---
194472024-02-06T22:23:55.754Z 24 480-499 177 177 177 322 322 322 T T F <---
194482024-02-06T22:23:55.755Z 25 500-519 177 177 177 322 322 322 F F F
194492024-02-06T22:23:55.755Z 26 520-539 176 176 176 319 319 319 F F F
194502024-02-06T22:23:55.755Z 27 540-559 172 172 172 313 313 313 T T F <---
194512024-02-06T22:23:55.755Z 28 560-579 172 172 172 314 314 314 T T F <---
194522024-02-06T22:23:55.755Z 29 580-599 169 169 169 307 307 307 F F F
194532024-02-06T22:23:55.755ZMax gen: 178, Max flush: 323
194542024-02-06T22:23:55.755ZError: Difference in extent metadata found!
194552024-02-06T22:23:55.755ZA Difference in extent metadata is expected here
194562024-02-06T22:23:55.755ZOn loop 89
194572024-02-06T22:23:55.755Z
194582024-02-06T22:23:55.758ZVerifying data now
194592024-02-06T22:24:00.317ZNo change after verify
194602024-02-06T22:24:00.317ZLoop: 89 Downstairs dump after verify (and repair):
194612024-02-06T22:24:00.330ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194622024-02-06T22:24:00.330Z 0 000-019 174 174 174 317 317 317 F F F
194632024-02-06T22:24:00.330Z 1 020-039 179 179 179 324 324 324 F F F
194642024-02-06T22:24:00.330Z 2 040-059 173 173 173 315 315 315 F F F
194652024-02-06T22:24:00.330Z 3 060-079 172 172 172 312 312 312 F F F
194662024-02-06T22:24:00.330Z 4 080-099 175 175 175 318 318 318 F F F
194672024-02-06T22:24:00.330Z 5 100-119 175 175 175 318 318 318 F F F
194682024-02-06T22:24:00.330Z 6 120-139 179 179 179 324 324 324 F F F
194692024-02-06T22:24:00.330Z 7 140-159 179 179 179 324 324 324 F F F
194702024-02-06T22:24:00.330Z 8 160-179 179 179 179 324 324 324 F F F
194712024-02-06T22:24:00.330Z 9 180-199 176 176 176 319 319 319 F F F
194722024-02-06T22:24:00.333Z 10 200-219 176 176 176 320 320 320 F F F
194732024-02-06T22:24:00.333Z 11 220-239 178 178 178 323 323 323 F F F
194742024-02-06T22:24:00.333Z 12 240-259 174 174 174 317 317 317 F F F
194752024-02-06T22:24:00.334Z 13 260-279 171 171 171 310 310 310 F F F
194762024-02-06T22:24:00.334Z 14 280-299 171 171 171 310 310 310 F F F
194772024-02-06T22:24:00.334Z 15 300-319 179 179 179 324 324 324 F F F
194782024-02-06T22:24:00.334Z 16 320-339 179 179 179 324 324 324 F F F
194792024-02-06T22:24:00.334Z 17 340-359 177 177 177 322 322 322 F F F
194802024-02-06T22:24:00.334Z 18 360-379 176 176 176 319 319 319 F F F
194812024-02-06T22:24:00.334Z 19 380-399 176 176 176 319 319 319 F F F
194822024-02-06T22:24:00.334Z 20 400-419 168 168 168 306 306 306 F F F
194832024-02-06T22:24:00.334Z 21 420-439 164 164 164 300 300 300 F F F
194842024-02-06T22:24:00.334Z 22 440-459 177 177 177 322 322 322 F F F
194852024-02-06T22:24:00.334Z 23 460-479 178 178 178 323 323 323 F F F
194862024-02-06T22:24:00.334Z 24 480-499 179 179 179 324 324 324 F F F
194872024-02-06T22:24:00.334Z 25 500-519 177 177 177 322 322 322 F F F
194882024-02-06T22:24:00.334Z 26 520-539 176 176 176 319 319 319 F F F
194892024-02-06T22:24:00.334Z 27 540-559 179 179 179 324 324 324 F F F
194902024-02-06T22:24:00.334Z 28 560-579 179 179 179 324 324 324 F F F
194912024-02-06T22:24:00.334Z 29 580-599 169 169 169 307 307 307 F F F
194922024-02-06T22:24:00.334ZMax gen: 179, Max flush: 324
194932024-02-06T22:24:00.334Z
194942024-02-06T22:24:00.334ZBegin loop 90
194952024-02-06T22:24:00.334ZDownstairs to restart: 0
194962024-02-06T22:24:00.354Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.524394564Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858}
194972024-02-06T22:24:00.354Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.524646949Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858}
194982024-02-06T22:24:00.354Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.524689833Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858}
194992024-02-06T22:24:00.367Z{"msg":"Crucible 6558d281-084a-4817-a2d5-f1537b2134ef has session id: 14def2a2-9f63-41da-947a-9064c846a632","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.52471591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195002024-02-06T22:24:00.367Z{"msg":"Upstairs opts: Upstairs UUID: 6558d281-084a-4817-a2d5-f1537b2134ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.524763714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195012024-02-06T22:24:00.367Z{"msg":"Crucible stats registered with UUID: 6558d281-084a-4817-a2d5-f1537b2134ef","v":0,"name":"crucible","level":30Crucible runtime is spawned
195022024-02-06T22:24:00.367Z,"time":"2024-02-06T22:23:59.524784151Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195032024-02-06T22:24:00.367Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.524886987Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858}
195042024-02-06T22:24:00.367Z{"msg":"6558d281-084a-4817-a2d5-f1537b2134ef active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525056134Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195052024-02-06T22:24:00.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.52508806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195062024-02-06T22:24:00.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525104588Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195072024-02-06T22:24:00.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525128004Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195082024-02-06T22:24:00.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525155561Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195092024-02-06T22:24:00.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525181877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195102024-02-06T22:24:00.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525204894Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195112024-02-06T22:24:00.368Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525242639Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195122024-02-06T22:24:00.368Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525275665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195132024-02-06T22:24:00.368Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525327678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195142024-02-06T22:24:00.368Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525357223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195152024-02-06T22:24:00.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525470848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195162024-02-06T22:24:00.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.525526591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195172024-02-06T22:24:00.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.526205119Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195182024-02-06T22:24:00.369Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.526254532Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195192024-02-06T22:24:00.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.526302276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195202024-02-06T22:24:00.369Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.526349269Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195212024-02-06T22:24:00.369Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.526870169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195222024-02-06T22:24:00.369Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.526918403Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195232024-02-06T22:24:00.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.52708381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195242024-02-06T22:24:00.369Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527123075Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195252024-02-06T22:24:00.369Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527469078Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195262024-02-06T22:24:00.369Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527506463Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195272024-02-06T22:24:00.369Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527542778Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195282024-02-06T22:24:00.370Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527580833Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195292024-02-06T22:24:00.370Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527608259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195302024-02-06T22:24:00.370Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527647574Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195312024-02-06T22:24:00.370Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.527911478Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195322024-02-06T22:24:00.370Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528354229Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195332024-02-06T22:24:00.370Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528396093Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195342024-02-06T22:24:00.370Z{"msg":"[0]R flush_numbers[0..12]: [317, 324, 315, 312, 318, 318, 324, 324, 324, 319, 320, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.52841291Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195352024-02-06T22:24:00.370Z{"msg":"[0]R generation[0..12]: [174, 179, 173, 172, 175, 175, 179, 179, 179, 176, 176, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528437817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195362024-02-06T22:24:00.370Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528453195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195372024-02-06T22:24:00.370Z{"msg":"[1]R flush_numbers[0..12]: [317, 324, 315, 312, 318, 318, 324, 324, 324, 319, 320, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528477192Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195382024-02-06T22:24:00.370Z{"msg":"[1]R generation[0..12]: [174, 179, 173, 172, 175, 175, 179, 179, 179, 176, 176, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528492Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195392024-02-06T22:24:00.370Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528515487Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195402024-02-06T22:24:00.370Z{"msg":"[2]R flush_numbers[0..12]: [317, 324, 315, 312, 318, 318, 324, 324, 324, 319, 320, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528531385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195412024-02-06T22:24:00.371Z{"msg":"[2]R generation[0..12]: [174, 179, 173, 172, 175, 175, 179, 179, 179, 176, 176, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.52856234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"Wait for a query_work_queue command to finish before sending IO
195422024-02-06T22:24:00.371Zdownstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195432024-02-06T22:24:00.371Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528604195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195442024-02-06T22:24:00.371Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.5286399Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195452024-02-06T22:24:00.371Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528664077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195462024-02-06T22:24:00.371Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528678005Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195472024-02-06T22:24:00.371Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528699452Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195482024-02-06T22:24:00.371Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.52871587Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195492024-02-06T22:24:00.371Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528748435Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195502024-02-06T22:24:00.372Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528783331Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195512024-02-06T22:24:00.372Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528819186Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195522024-02-06T22:24:00.372Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528855591Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195532024-02-06T22:24:00.372Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528880298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195542024-02-06T22:24:00.372Z{"msg":"6558d281-084a-4817-a2d5-f1537b2134ef is now active with session: 14def2a2-9f63-41da-947a-9064c846a632","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528894036Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195552024-02-06T22:24:00.372Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528921142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858,"session_id":"14def2a2-9f63-41da-947a-9064c846a632"}
195562024-02-06T22:24:00.372Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:23:59.528957087Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858}
195572024-02-06T22:24:00.372ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195582024-02-06T22:24:00.372ZLoading write count information from file "/tmp/test_repair_verify.data"
195592024-02-06T22:24:00.372ZRun Repair workload
195602024-02-06T22:24:00.379Z01/30 Write block 180 len 32768 data: 19 21 19 22 23 22 22 21
195612024-02-06T22:24:00.379Z02/30 Read block 254 len 32768
195622024-02-06T22:24:00.379Z03/30 Read block 345 len 24576
195632024-02-06T22:24:00.379Z04/30 Read block 511 len 20480
195642024-02-06T22:24:00.380Z05/30 Write block 30 len 12288 data: 14 11 9
195652024-02-06T22:24:00.380Z06/30 Read block 301 len 28672
195662024-02-06T22:24:00.380Z07/30 Read block 216 len 32768
195672024-02-06T22:24:00.380Z08/30 Write block 265 len 32768 data: 23 21 20 18 16 14 13 14
195682024-02-06T22:24:00.380Z09/30 Write block 296 len 28672 data: 14 10 10 12 15 15 16
195692024-02-06T22:24:00.380Z10/30 Read block 129 len 28672
195702024-02-06T22:24:00.380Z11/30 Flush
195712024-02-06T22:24:00.380Z12/30 Write block 518 len 32768 data: 12 6 8 9 9 8 8 8
195722024-02-06T22:24:00.380Z13/30 Write block 34 len 40960 data: 9 8 9 10 12 11 15 14 11 11
195732024-02-06T22:24:00.380Z14/30 Read block 299 len 12288
195742024-02-06T22:24:00.381Z15/30 Read block 205 len 28672
195752024-02-06T22:24:00.381Z16/30 Write block 420 len 20480 data: 17 15 14 12 13
195762024-02-06T22:24:00.386Z17/30 Write block 182 len 12288 data: 20 23 24
195772024-02-06T22:24:00.386Z18/30 Read block 387 len 28672
195782024-02-06T22:24:00.386Z19/30 Read block 331 len 4096
195792024-02-06T22:24:00.386Z20/30 Write block 277 len 8192 data: 12 17
195802024-02-06T22:24:00.387Z21/30 Read block 60 len 16384
195812024-02-06T22:24:00.387Z22/30 Write block 283 len 24576 data: 12 13 11 9 8 13
195822024-02-06T22:24:00.387Z23/30 Write block 45 len 20480 data: 11 12 13 14 16
195832024-02-06T22:24:00.387Z24/30 Write block 17 len 8192 data: 13 13
195842024-02-06T22:24:00.387Z25/30 Write block 435 len 36864 data: 10 13 15 13 15 12 9 11 10
195852024-02-06T22:24:00.387Z26/30 Write block 569 len 40960 data: 14 14 19 19 18 20 16 15 16 17
195862024-02-06T22:24:00.387Z27/30 Read block 493 len 32768
195872024-02-06T22:24:00.391Z28/30 Read block 455 len 12288
195882024-02-06T22:24:00.391Z29/30 Read block 563 len 16384
195892024-02-06T22:24:00.391Z30/30 Write block 166 len 28672 data: 14 17 20 19 19 14 15
195902024-02-06T22:24:00.391Z----------------------------------------------------------------
195912024-02-06T22:24:00.391Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
195922024-02-06T22:24:00.391ZStates: Active Active Active
195932024-02-06T22:24:00.391ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195942024-02-06T22:24:00.391Z 1 Acked 1000 Write 8 Sent Done Done false
195952024-02-06T22:24:00.391Z 2 Acked 1001 Read 8 Sent Done Done false
195962024-02-06T22:24:00.391Z 3 Acked 1002 Read 6 Sent Done Done false
195972024-02-06T22:24:00.392Z 4 Acked 1003 Read 5 Sent Done Done false
195982024-02-06T22:24:00.392Z 5 Acked 1004 Write 3 Sent Done Done false
195992024-02-06T22:24:00.392Z 6 Acked 1005 Read 7 Sent Done Done false
196002024-02-06T22:24:00.392Z 7 Acked 1006 Read 8 Sent Done Done false
196012024-02-06T22:24:00.392Z 8 Acked 1007 Write 8 Sent Done Done false
196022024-02-06T22:24:00.392Z 9 Acked 1008 Write 7 Sent Done Done false
196032024-02-06T22:24:00.392Z 10 Acked 1009 Read 7 Sent Done Done false
196042024-02-06T22:24:00.392Z 11 Acked 1010 Flush 0 Sent Done Done false
196052024-02-06T22:24:00.392Z 12 Acked 1011 Write 8 Sent Done Done false
196062024-02-06T22:24:00.392Z 13 Acked 1012 Write 10 Sent Done Done false
196072024-02-06T22:24:00.392Z 14 Acked 1013 Read 3 Sent Done Done false
196082024-02-06T22:24:00.392Z 15 Acked 1014 Read 7 Sent Done Done false
196092024-02-06T22:24:00.392Z 16 Acked 1015 Write 5 Sent Done Done false
196102024-02-06T22:24:00.392Z 17 Acked 1016 Write 3 Sent Done Done false
196112024-02-06T22:24:00.392Z 18 Acked 1017 Read 7 Sent Done Done false
196122024-02-06T22:24:00.392Z 19 Acked 1018 Read 1 Sent Done Done false
196132024-02-06T22:24:00.392Z 20 Acked 1019 Write 2 Sent Done Done false
196142024-02-06T22:24:00.392Z 21 Acked 1020 Read 4 Sent Done Done false
196152024-02-06T22:24:00.392Z 22 Acked 1021 Write 6 Sent Done Done false
196162024-02-06T22:24:00.392Z 23 Acked 1022 Write 5 Sent Done Done false
196172024-02-06T22:24:00.392Z 24 Acked 1023 Write 2 Sent Done Done false
196182024-02-06T22:24:00.392Z 25 Acked 1024 Write 9 Sent Done Done false
196192024-02-06T22:24:00.392Z 26 Acked 1025 Write 10 Sent Done Done false
196202024-02-06T22:24:00.392Z 27 Acked 1026 Read 8 Sent Done Done false
196212024-02-06T22:24:00.393Z 28 Acked 1027 Read 3 Sent Done Done false
196222024-02-06T22:24:00.393Z 29 Acked 1028 Read 4 Sent Done Done false
196232024-02-06T22:24:00.393Z 30 Acked 1029 Write 7 Sent Sent Sent false
196242024-02-06T22:24:00.393Z STATES DS:0 DS:1 DS:2 TOTAL
196252024-02-06T22:24:00.393Z New 0 0 0 0
196262024-02-06T22:24:00.393Z Sent 30 1 1 32
196272024-02-06T22:24:00.393Z Done 0 29 29 58
196282024-02-06T22:24:00.393Z Skipped 0 0 0 0
196292024-02-06T22:24:00.393Z Error 0 0 0 0
196302024-02-06T22:24:00.393ZLast Flush: 0 1010 1010
196312024-02-06T22:24:00.393ZDownstairs last five completed:
196322024-02-06T22:24:00.393ZUpstairs last five completed: 30 29 28 27 26
196332024-02-06T22:24:00.393Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:23:59.561078296Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1858}
196342024-02-06T22:24:00.393ZWrote out file "/tmp/test_repair_verify.data"
196352024-02-06T22:24:00.396Z
196362024-02-06T22:24:00.399ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196372024-02-06T22:24:00.418ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196382024-02-06T22:24:00.418Z 0 000-019 174 174 174 317 317 317 F T T <---
196392024-02-06T22:24:00.418Z 1 020-039 179 180 180 324 325 325 F T T <---
196402024-02-06T22:24:00.418Z 2 040-059 173 173 173 315 315 315 F T T <---
196412024-02-06T22:24:00.418Z 3 060-079 172 172 172 312 312 312 F F F
196422024-02-06T22:24:00.418Z 4 080-099 175 175 175 318 318 318 F F F
196432024-02-06T22:24:00.418Z 5 100-119 175 175 175 318 318 318 F F F
196442024-02-06T22:24:00.418Z 6 120-139 179 179 179 324 324 324 F F F
196452024-02-06T22:24:00.422Z 7 140-159 179 179 179 324 324 324 F F F
196462024-02-06T22:24:00.422Z 8 160-179 179 179 179 324 324 324 F T T <---
196472024-02-06T22:24:00.422Z 9 180-199 176 180 180 319 325 325 F T T <---
196482024-02-06T22:24:00.422Z 10 200-219 176 176 176 320 320 320 F F F
196492024-02-06T22:24:00.422Z 11 220-239 178 178 178 323 323 323 F F F
196502024-02-06T22:24:00.422Z 12 240-259 174 174 174 317 317 317 F F F
196512024-02-06T22:24:00.422Z 13 260-279 171 180 180 310 325 325 F T T <---
196522024-02-06T22:24:00.422Z 14 280-299 171 180 180 310 325 325 F T T <---
196532024-02-06T22:24:00.422Z 15 300-319 179 180 180 324 325 325 F F F <---
196542024-02-06T22:24:00.422Z 16 320-339 179 179 179 324 324 324 F F F
196552024-02-06T22:24:00.422Z 17 340-359 177 177 177 322 322 322 F F F
196562024-02-06T22:24:00.422Z 18 360-379 176 176 176 319 319 319 F F F
196572024-02-06T22:24:00.422Z 19 380-399 176 176 176 319 319 319 F F F
196582024-02-06T22:24:00.422Z 20 400-419 168 168 168 306 306 306 F F F
196592024-02-06T22:24:00.422Z 21 420-439 164 164 164 300 300 300 F T T <---
196602024-02-06T22:24:00.422Z 22 440-459 177 177 177 322 322 322 F T T <---
196612024-02-06T22:24:00.422Z 23 460-479 178 178 178 323 323 323 F F F
196622024-02-06T22:24:00.422Z 24 480-499 179 179 179 324 324 324 F F F
196632024-02-06T22:24:00.422Z 25 500-519 177 177 177 322 322 322 F T T <---
196642024-02-06T22:24:00.422Z 26 520-539 176 176 176 319 319 319 F T T <---
196652024-02-06T22:24:00.422Z 27 540-559 179 179 179 324 324 324 F F F
196662024-02-06T22:24:00.423Z 28 560-579 179 179 179 324 324 324 F T T <---
196672024-02-06T22:24:00.423Z 29 580-599 169 169 169 307 307 307 F F F
196682024-02-06T22:24:00.423ZMax gen: 180, Max flush: 325
196692024-02-06T22:24:00.423ZError: Difference in extent metadata found!
196702024-02-06T22:24:00.423ZA Difference in extent metadata is expected here
196712024-02-06T22:24:00.423ZOn loop 90
196722024-02-06T22:24:00.423Z
196732024-02-06T22:24:00.426ZVerifying data now
196742024-02-06T22:24:05.076ZNo change after verify
196752024-02-06T22:24:05.076ZLoop: 90 Downstairs dump after verify (and repair):
196762024-02-06T22:24:05.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196772024-02-06T22:24:05.091Z 0 000-019 181 181 181 326 326 326 F F F
196782024-02-06T22:24:05.091Z 1 020-039 181 181 181 326 326 326 F F F
196792024-02-06T22:24:05.091Z 2 040-059 181 181 181 326 326 326 F F F
196802024-02-06T22:24:05.091Z 3 060-079 172 172 172 312 312 312 F F F
196812024-02-06T22:24:05.092Z 4 080-099 175 175 175 318 318 318 F F F
196822024-02-06T22:24:05.092Z 5 100-119 175 175 175 318 318 318 F F F
196832024-02-06T22:24:05.092Z 6 120-139 179 179 179 324 324 324 F F F
196842024-02-06T22:24:05.092Z 7 140-159 179 179 179 324 324 324 F F F
196852024-02-06T22:24:05.092Z 8 160-179 181 181 181 326 326 326 F F F
196862024-02-06T22:24:05.092Z 9 180-199 181 181 181 326 326 326 F F F
196872024-02-06T22:24:05.092Z 10 200-219 176 176 176 320 320 320 F F F
196882024-02-06T22:24:05.092Z 11 220-239 178 178 178 323 323 323 F F F
196892024-02-06T22:24:05.100Z 12 240-259 174 174 174 317 317 317 F F F
196902024-02-06T22:24:05.100Z 13 260-279 181 181 181 326 326 326 F F F
196912024-02-06T22:24:05.100Z 14 280-299 181 181 181 326 326 326 F F F
196922024-02-06T22:24:05.100Z 15 300-319 180 180 180 325 325 325 F F F
196932024-02-06T22:24:05.100Z 16 320-339 179 179 179 324 324 324 F F F
196942024-02-06T22:24:05.100Z 17 340-359 177 177 177 322 322 322 F F F
196952024-02-06T22:24:05.100Z 18 360-379 176 176 176 319 319 319 F F F
196962024-02-06T22:24:05.100Z 19 380-399 176 176 176 319 319 319 F F F
196972024-02-06T22:24:05.100Z 20 400-419 168 168 168 306 306 306 F F F
196982024-02-06T22:24:05.100Z 21 420-439 181 181 181 326 326 326 F F F
196992024-02-06T22:24:05.100Z 22 440-459 181 181 181 326 326 326 F F F
197002024-02-06T22:24:05.101Z 23 460-479 178 178 178 323 323 323 F F F
197012024-02-06T22:24:05.101Z 24 480-499 179 179 179 324 324 324 F F F
197022024-02-06T22:24:05.101Z 25 500-519 181 181 181 326 326 326 F F F
197032024-02-06T22:24:05.101Z 26 520-539 181 181 181 326 326 326 F F F
197042024-02-06T22:24:05.101Z 27 540-559 179 179 179 324 324 324 F F F
197052024-02-06T22:24:05.101Z 28 560-579 181 181 181 326 326 326 F F F
197062024-02-06T22:24:05.101Z 29 580-599 169 169 169 307 307 307 F F F
197072024-02-06T22:24:05.101ZMax gen: 181, Max flush: 326
197082024-02-06T22:24:05.101Z
197092024-02-06T22:24:05.101ZBegin loop 91
197102024-02-06T22:24:05.101ZDownstairs to restart: 2
197112024-02-06T22:24:05.116Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282275795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866}
197122024-02-06T22:24:05.116Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282521292Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866}
197132024-02-06T22:24:05.116Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282548238Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866}
197142024-02-06T22:24:05.127Z{"msg":"Crucible e5cecd37-558d-4243-9631-ab9d5fff973d has session id: 79644db2-5c73-4416-b042-e7b47a871a89","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282582794Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866Crucible runtime is spawned
197152024-02-06T22:24:05.127Z,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197162024-02-06T22:24:05.127Z{"msg":"Upstairs opts: Upstairs UUID: e5cecd37-558d-4243-9631-ab9d5fff973d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.28268161Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197172024-02-06T22:24:05.127Z{"msg":"Crucible stats registered with UUID: e5cecd37-558d-4243-9631-ab9d5fff973d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282725134Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197182024-02-06T22:24:05.127Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282748761Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866}
197192024-02-06T22:24:05.127Z{"msg":"e5cecd37-558d-4243-9631-ab9d5fff973d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282781287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197202024-02-06T22:24:05.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282817822Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197212024-02-06T22:24:05.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282854897Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197222024-02-06T22:24:05.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282880463Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197232024-02-06T22:24:05.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.28290558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197242024-02-06T22:24:05.128Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282932586Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197252024-02-06T22:24:05.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.282960902Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197262024-02-06T22:24:05.128Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283000877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197272024-02-06T22:24:05.128Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283032833Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197282024-02-06T22:24:05.128Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.28305663Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197292024-02-06T22:24:05.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283090735Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197302024-02-06T22:24:05.128Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:04.283169514Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197312024-02-06T22:24:05.128Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:04.283213308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197322024-02-06T22:24:05.128Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283257182Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197332024-02-06T22:24:05.129Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:04.283286978Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197342024-02-06T22:24:05.129Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283317434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197352024-02-06T22:24:05.129Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283337992Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197362024-02-06T22:24:05.129Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283490511Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197372024-02-06T22:24:05.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283751276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197382024-02-06T22:24:05.129Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283787471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197392024-02-06T22:24:05.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.28386823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197402024-02-06T22:24:05.129Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.283954808Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197412024-02-06T22:24:05.129Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.284296272Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197422024-02-06T22:24:05.130Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.284413756Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197432024-02-06T22:24:05.130Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.284807283Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197442024-02-06T22:24:05.130Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.284866235Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197452024-02-06T22:24:05.130Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.284918308Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197462024-02-06T22:24:05.130Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.284936925Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197472024-02-06T22:24:05.130Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.284960212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197482024-02-06T22:24:05.130Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:04.28497552Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197492024-02-06T22:24:06.114Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.284549975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197502024-02-06T22:24:06.120Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.284907917Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197512024-02-06T22:24:06.120Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.285567938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197522024-02-06T22:24:06.120Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.285612442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197532024-02-06T22:24:06.120Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286052182Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197542024-02-06T22:24:06.120Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286542136Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197552024-02-06T22:24:06.120Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286577262Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197562024-02-06T22:24:06.120Z{"msg":"[0]R flush_numbers[0..12]: [326, 326, 326, 312, 318, 318, 324, 324, 326, 326, 320, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286596219Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197572024-02-06T22:24:06.120Z{"msg":"[0]R generation[0..12]: [181, 181, 181, 172, 175, 175, 179, 179, 181, 181, 176, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286636123Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197582024-02-06T22:24:06.120Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286665839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197592024-02-06T22:24:06.121Z{"msg":"[1]R flush_numbers[0..12]: [326, 326, 326, 312, 318, 318, 324, 324, 326, 326, 320, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286702585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197602024-02-06T22:24:06.121Z{"msg":"[1]R generation[0..12]: [181, 181, 181, 172, 175, 175, 179, 179, 181, 181, 176, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286723572Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197612024-02-06T22:24:06.121ZWait for a query_work_queue command to finish before sending IO
197622024-02-06T22:24:06.121Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286768516Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197632024-02-06T22:24:06.121Z{"msg":"[2]R flush_numbers[0..12]: [326, 326, 326, 312, 318, 318, 324, 324, 326, 326, 320, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286794972Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197642024-02-06T22:24:06.121Z{"msg":"[2]R generation[0..12]: [181, 181, 181, 172, 175, 175, 179, 179, 181, 181, 176, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286832077Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197652024-02-06T22:24:06.121Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286852744Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197662024-02-06T22:24:06.121Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286889709Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197672024-02-06T22:24:06.121Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286909997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197682024-02-06T22:24:06.121Z{"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286947001Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197692024-02-06T22:24:06.121Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.286965839Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197702024-02-06T22:24:06.121Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.287003814Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197712024-02-06T22:24:06.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.287022651Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197722024-02-06T22:24:06.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.287059926Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197732024-02-06T22:24:06.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.287076924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197742024-02-06T22:24:06.121Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.28710768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197752024-02-06T22:24:06.121Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.287122778Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197762024-02-06T22:24:06.121Z{"msg":"e5cecd37-558d-4243-9631-ab9d5fff973d is now active with session: 79644db2-5c73-4416-b042-e7b47a871a89","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197772024-02-06T22:24:06.121Z,"time":"2024-02-06T22:24:05.287153444Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197782024-02-06T22:24:06.121Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.287170721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866,"session_id":"79644db2-5c73-4416-b042-e7b47a871a89"}
197792024-02-06T22:24:06.121Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:05.287200007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866}
197802024-02-06T22:24:06.121ZLoading write count information from file "/tmp/test_repair_verify.data"
197812024-02-06T22:24:06.121ZRun Repair workload
197822024-02-06T22:24:06.121Z01/30 Write block 230 len 40960 data: 13 16 17 14 17 18 17 15 10 10
197832024-02-06T22:24:06.128Z02/30 Read block 182 len 8192
197842024-02-06T22:24:06.128Z03/30 Write block 99 len 40960 data: 15 13 9 9 7 6 9 13 15 18
197852024-02-06T22:24:06.128Z04/30 Write block 412 len 40960 data: 14 13 12 15 15 17 17 17 18 16
197862024-02-06T22:24:06.129Z05/30 Write block 392 len 36864 data: 10 10 9 8 7 8 10 11 12
197872024-02-06T22:24:06.129Z06/30 Read block 441 len 4096
197882024-02-06T22:24:06.129Z07/30 Read block 334 len 32768
197892024-02-06T22:24:06.129Z08/30 Read block 92 len 12288
197902024-02-06T22:24:06.129Z09/30 Read block 526 len 36864
197912024-02-06T22:24:06.129Z10/30 Write block 53 len 12288 data: 25 29 26
197922024-02-06T22:24:06.129Z11/30 Write block 527 len 20480 data: 11 11 16 18 19
197932024-02-06T22:24:06.129Z12/30 Read block 366 len 8192
197942024-02-06T22:24:06.132Z13/30 Write block 42 len 16384 data: 12 12 11 12
197952024-02-06T22:24:06.132Z14/30 Write block 364 len 8192 data: 14 14
197962024-02-06T22:24:06.132Z15/30 Read block 409 len 8192
197972024-02-06T22:24:06.132Z16/30 Write block 380 len 8192 data: 14 15
197982024-02-06T22:24:06.132Z17/30 Read block 397 len 28672
197992024-02-06T22:24:06.132Z18/30 Read block 176 len 4096
198002024-02-06T22:24:06.135Z19/30 Read block 181 len 36864
198012024-02-06T22:24:06.135Z20/30 Read block 99 len 28672
198022024-02-06T22:24:06.135Z21/30 Write block 80 len 16384 data: 8 9 12 12
198032024-02-06T22:24:06.135Z22/30 Read block 189 len 40960
198042024-02-06T22:24:06.140Z23/30 Read block 549 len 36864
198052024-02-06T22:24:06.140Z24/30 Flush
198062024-02-06T22:24:06.143Z25/30 Write block 512 len 28672 data: 13 17 18 15 13 15 13
198072024-02-06T22:24:06.143Z26/30 Read block 375 len 20480
198082024-02-06T22:24:06.143Z27/30 Write block 588 len 20480 data: 15 18 22 16 14
198092024-02-06T22:24:06.148Z28/30 Write block 587 len 12288 data: 15 16 19
198102024-02-06T22:24:06.148Z29/30 Read block 499 len 16384
198112024-02-06T22:24:06.148Z30/30 Write block 315 len 16384 data: 20 19 18 17
198122024-02-06T22:24:06.148Z----------------------------------------------------------------
198132024-02-06T22:24:06.148Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
198142024-02-06T22:24:06.148ZStates: Active Active Active
198152024-02-06T22:24:06.148ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198162024-02-06T22:24:06.148Z 1 Acked 1000 Write 10 Done Done Sent false
198172024-02-06T22:24:06.149Z 2 Acked 1001 Read 2 Done Done Sent false
198182024-02-06T22:24:06.149Z 3 Acked 1002 Write 10 Done Done Sent false
198192024-02-06T22:24:06.149Z 4 Acked 1003 Write 10 Done Done Sent false
198202024-02-06T22:24:06.149Z 5 Acked 1004 Write 9 Done Done Sent false
198212024-02-06T22:24:06.149Z 6 Acked 1005 Read 1 Done Done Sent false
198222024-02-06T22:24:06.149Z 7 Acked 1006 Read 8 Done Done Sent false
198232024-02-06T22:24:06.149Z 8 Acked 1007 Read 3 Done Done Sent false
198242024-02-06T22:24:06.149Z 9 Acked 1008 Read 9 Done Done Sent false
198252024-02-06T22:24:06.149Z 10 Acked 1009 Write 3 Done Done Sent false
198262024-02-06T22:24:06.149Z 11 Acked 1010 Write 5 Done Done Sent false
198272024-02-06T22:24:06.149Z 12 Acked 1011 Read 2 Done Done Sent false
198282024-02-06T22:24:06.149Z 13 Acked 1012 Write 4 Done Done Sent false
198292024-02-06T22:24:06.149Z 14 Acked 1013 Write 2 Done Done Sent false
198302024-02-06T22:24:06.149Z 15 Acked 1014 Read 2 Done Done Sent false
198312024-02-06T22:24:06.149Z 16 Acked 1015 Write 2 Done Done Sent false
198322024-02-06T22:24:06.149Z 17 Acked 1016 Read 7 Done Done Sent false
198332024-02-06T22:24:06.149Z 18 Acked 1017 Read 1 Done Done Sent false
198342024-02-06T22:24:06.149Z 19 Acked 1018 Read 9 Done Done Sent false
198352024-02-06T22:24:06.149Z 20 Acked 1019 Read 7 Done Done Sent false
198362024-02-06T22:24:06.149Z 21 Acked 1020 Write 4 Done Done Sent false
198372024-02-06T22:24:06.149Z 22 Acked 1021 Read 10 Done Done Sent false
198382024-02-06T22:24:06.149Z 23 Acked 1022 Read 9 Done Done Sent false
198392024-02-06T22:24:06.149Z 24 Acked 1023 Flush 0 Done Done Sent false
198402024-02-06T22:24:06.149Z 25 Acked 1024 Write 7 Done Done Sent false
198412024-02-06T22:24:06.149Z 26 Acked 1025 Read 5 Done Done Sent false
198422024-02-06T22:24:06.150Z 27 Acked 1026 Write 5 Done Done Sent false
198432024-02-06T22:24:06.150Z 28 Acked 1027 Write 3 Done Done Sent false
198442024-02-06T22:24:06.150Z 29 Acked 1028 Read 4 Done Done Sent false
198452024-02-06T22:24:06.150Z 30 Acked 1029 Write 4 Sent Sent Sent false
198462024-02-06T22:24:06.150Z STATES DS:0 DS:1 DS:2 TOTAL
198472024-02-06T22:24:06.150Z New 0 0 0 0
198482024-02-06T22:24:06.150Z Sent 1 1 30 32
198492024-02-06T22:24:06.150Z Done 29 29 0 58
198502024-02-06T22:24:06.150Z Skipped 0 0 0 0
198512024-02-06T22:24:06.150Z Error 0 0 0 0
198522024-02-06T22:24:06.150ZLast Flush: 1023 1023 0
198532024-02-06T22:24:06.150ZDownstairs last five completed:
198542024-02-06T22:24:06.150ZUpstairs last five completed: 30 29 28 27 26
198552024-02-06T22:24:06.150Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:05.316728997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1866}
198562024-02-06T22:24:06.150ZWrote out file "/tmp/test_repair_verify.data"
198572024-02-06T22:24:06.154Z
198582024-02-06T22:24:06.157ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198592024-02-06T22:24:06.171ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198602024-02-06T22:24:06.171Z 0 000-019 181 181 181 326 326 326 F F F
198612024-02-06T22:24:06.171Z 1 020-039 181 181 181 326 326 326 F F F
198622024-02-06T22:24:06.171Z 2 040-059 182 182 181 327 327 326 F F F <---
198632024-02-06T22:24:06.171Z 3 060-079 172 172 172 312 312 312 F F F
198642024-02-06T22:24:06.171Z 4 080-099 182 182 175 327 327 318 F F F <---
198652024-02-06T22:24:06.171Z 5 100-119 182 182 175 327 327 318 F F F <---
198662024-02-06T22:24:06.171Z 6 120-139 179 179 179 324 324 324 F F F
198672024-02-06T22:24:06.171Z 7 140-159 179 179 179 324 324 324 F F F
198682024-02-06T22:24:06.171Z 8 160-179 181 181 181 326 326 326 F F F
198692024-02-06T22:24:06.175Z 9 180-199 181 181 181 326 326 326 F F F
198702024-02-06T22:24:06.175Z 10 200-219 176 176 176 320 320 320 F F F
198712024-02-06T22:24:06.175Z 11 220-239 182 182 178 327 327 323 F F F <---
198722024-02-06T22:24:06.175Z 12 240-259 174 174 174 317 317 317 F F F
198732024-02-06T22:24:06.175Z 13 260-279 181 181 181 326 326 326 F F F
198742024-02-06T22:24:06.175Z 14 280-299 181 181 181 326 326 326 F F F
198752024-02-06T22:24:06.175Z 15 300-319 180 180 180 325 325 325 T T F <---
198762024-02-06T22:24:06.175Z 16 320-339 179 179 179 324 324 324 F F F
198772024-02-06T22:24:06.175Z 17 340-359 177 177 177 322 322 322 F F F
198782024-02-06T22:24:06.175Z 18 360-379 182 182 176 327 327 319 F F F <---
198792024-02-06T22:24:06.175Z 19 380-399 182 182 176 327 327 319 F F F <---
198802024-02-06T22:24:06.175Z 20 400-419 182 182 168 327 327 306 F F F <---
198812024-02-06T22:24:06.175Z 21 420-439 182 182 181 327 327 326 F F F <---
198822024-02-06T22:24:06.175Z 22 440-459 181 181 181 326 326 326 F F F
198832024-02-06T22:24:06.175Z 23 460-479 178 178 178 323 323 323 F F F
198842024-02-06T22:24:06.176Z 24 480-499 179 179 179 324 324 324 F F F
198852024-02-06T22:24:06.176Z 25 500-519 181 181 181 326 326 326 T T F <---
198862024-02-06T22:24:06.176Z 26 520-539 182 182 181 327 327 326 F F F <---
198872024-02-06T22:24:06.176Z 27 540-559 179 179 179 324 324 324 F F F
198882024-02-06T22:24:06.176Z 28 560-579 181 181 181 326 326 326 F F F
198892024-02-06T22:24:06.176Z 29 580-599 169 169 169 307 307 307 T T F <---
198902024-02-06T22:24:06.176ZMax gen: 182, Max flush: 327
198912024-02-06T22:24:06.176ZError: Difference in extent metadata found!
198922024-02-06T22:24:06.176ZA Difference in extent metadata is expected here
198932024-02-06T22:24:06.176ZOn loop 91
198942024-02-06T22:24:06.176Z
198952024-02-06T22:24:06.179ZVerifying data now
198962024-02-06T22:24:10.774ZNo change after verify
198972024-02-06T22:24:10.774ZLoop: 91 Downstairs dump after verify (and repair):
198982024-02-06T22:24:10.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198992024-02-06T22:24:10.789Z 0 000-019 181 181 181 326 326 326 F F F
199002024-02-06T22:24:10.789Z 1 020-039 181 181 181 326 326 326 F F F
199012024-02-06T22:24:10.789Z 2 040-059 182 182 182 327 327 327 F F F
199022024-02-06T22:24:10.789Z 3 060-079 172 172 172 312 312 312 F F F
199032024-02-06T22:24:10.789Z 4 080-099 182 182 182 327 327 327 F F F
199042024-02-06T22:24:10.789Z 5 100-119 182 182 182 327 327 327 F F F
199052024-02-06T22:24:10.789Z 6 120-139 179 179 179 324 324 324 F F F
199062024-02-06T22:24:10.789Z 7 140-159 179 179 179 324 324 324 F F F
199072024-02-06T22:24:10.798Z 8 160-179 181 181 181 326 326 326 F F F
199082024-02-06T22:24:10.798Z 9 180-199 181 181 181 326 326 326 F F F
199092024-02-06T22:24:10.799Z 10 200-219 176 176 176 320 320 320 F F F
199102024-02-06T22:24:10.799Z 11 220-239 182 182 182 327 327 327 F F F
199112024-02-06T22:24:10.799Z 12 240-259 174 174 174 317 317 317 F F F
199122024-02-06T22:24:10.799Z 13 260-279 181 181 181 326 326 326 F F F
199132024-02-06T22:24:10.799Z 14 280-299 181 181 181 326 326 326 F F F
199142024-02-06T22:24:10.799Z 15 300-319 183 183 183 328 328 328 F F F
199152024-02-06T22:24:10.799Z 16 320-339 179 179 179 324 324 324 F F F
199162024-02-06T22:24:10.799Z 17 340-359 177 177 177 322 322 322 F F F
199172024-02-06T22:24:10.799Z 18 360-379 182 182 182 327 327 327 F F F
199182024-02-06T22:24:10.800Z 19 380-399 182 182 182 327 327 327 F F F
199192024-02-06T22:24:10.800Z 20 400-419 182 182 182 327 327 327 F F F
199202024-02-06T22:24:10.800Z 21 420-439 182 182 182 327 327 327 F F F
199212024-02-06T22:24:10.800Z 22 440-459 181 181 181 326 326 326 F F F
199222024-02-06T22:24:10.800Z 23 460-479 178 178 178 323 323 323 F F F
199232024-02-06T22:24:10.800Z 24 480-499 179 179 179 324 324 324 F F F
199242024-02-06T22:24:10.800Z 25 500-519 183 183 183 328 328 328 F F F
199252024-02-06T22:24:10.800Z 26 520-539 182 182 182 327 327 327 F F F
199262024-02-06T22:24:10.800Z 27 540-559 179 179 179 324 324 324 F F F
199272024-02-06T22:24:10.800Z 28 560-579 181 181 181 326 326 326 F F F
199282024-02-06T22:24:10.800Z 29 580-599 183 183 183 328 328 328 F F F
199292024-02-06T22:24:10.800ZMax gen: 183, Max flush: 328
199302024-02-06T22:24:10.801Z
199312024-02-06T22:24:10.801ZBegin loop 92
199322024-02-06T22:24:10.801ZDownstairs to restart: 0
199332024-02-06T22:24:10.814Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.977604167Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874}
199342024-02-06T22:24:10.814Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.977858033Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874}
199352024-02-06T22:24:10.814Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.97787706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874}
199362024-02-06T22:24:10.830Z{"msg":"Crucible b029f422-97f4-4220-a420-4ecef717e1fa has session id: d63f7496-63dd-4801-8be0-7ad9a776d240","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.977921465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199372024-02-06T22:24:10.830ZCrucible runtime is spawned
199382024-02-06T22:24:10.830Z{"msg":"Upstairs opts: Upstairs UUID: b029f422-97f4-4220-a420-4ecef717e1fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.977977237Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199392024-02-06T22:24:10.830Z{"msg":"Crucible stats registered with UUID: b029f422-97f4-4220-a420-4ecef717e1fa","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978020501Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199402024-02-06T22:24:10.831Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978042768Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874}
199412024-02-06T22:24:10.831Z{"msg":"b029f422-97f4-4220-a420-4ecef717e1fa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978160062Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199422024-02-06T22:24:10.831Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978204856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199432024-02-06T22:24:10.831Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978227933Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199442024-02-06T22:24:10.831Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978277696Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199452024-02-06T22:24:10.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978332699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199462024-02-06T22:24:10.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978369274Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199472024-02-06T22:24:10.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978412048Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199482024-02-06T22:24:10.832Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.9785434Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199492024-02-06T22:24:10.832Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978606522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199502024-02-06T22:24:10.832Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978638737Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199512024-02-06T22:24:10.832Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978665684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199522024-02-06T22:24:10.832Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.97869472Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199532024-02-06T22:24:10.832Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.978744263Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199542024-02-06T22:24:10.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.979232417Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199552024-02-06T22:24:10.833Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.979278011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199562024-02-06T22:24:10.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.979306967Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199572024-02-06T22:24:10.833Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.979356171Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199582024-02-06T22:24:10.833Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.979858513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199592024-02-06T22:24:10.833Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.979898177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199602024-02-06T22:24:10.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.980496697Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199612024-02-06T22:24:10.834Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.980578605Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199622024-02-06T22:24:10.834Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.98061901Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199632024-02-06T22:24:10.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.980635208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199642024-02-06T22:24:10.834Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.980660024Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199652024-02-06T22:24:10.834Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.98069667Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199662024-02-06T22:24:10.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.980733934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199672024-02-06T22:24:10.834Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.980772059Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199682024-02-06T22:24:10.835Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981184133Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199692024-02-06T22:24:10.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981599877Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199702024-02-06T22:24:10.835Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981640072Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199712024-02-06T22:24:10.835Z{"msg":"[0]R flush_numbers[0..12]: [326, 326, 327, 312, 327, 327, 324, 324, 326, 326, 320, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981659109Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199722024-02-06T22:24:10.835Z{"msg":"[0]R generation[0..12]: [181, 181, 182, 172, 182, 182, 179, 179, 181, 181, 176, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981681506Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199732024-02-06T22:24:10.835Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981696714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199742024-02-06T22:24:10.836Z{"msg":"[1]R flush_numbers[0..12]: [326, 326, 327, 312, 327, 327, 324, 324, 326, 326, 320, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981718252Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199752024-02-06T22:24:10.836Z{"msg":"[1]R generation[0..12]: [181, 181, 182, 172, 182, 182, 179, 179, 181, 181, 176, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.98173287Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199762024-02-06T22:24:10.836Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981753927Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199772024-02-06T22:24:10.836Z{"msg":"[2]R flush_numbers[0..12]: [326, 326, 327, 312, 327, 327, 324, 324, 326, 326, 320, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981768645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199782024-02-06T22:24:10.836Z{"msg":"[2]R generation[0..12]: [181, 181, 182, 172, 182, 182, 179, 179, 181, 181, 176, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981789532Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199792024-02-06T22:24:10.836Z{"msg":"Wait for a query_work_queue command to finish before sending IO
199802024-02-06T22:24:10.836Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981806229Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199812024-02-06T22:24:10.837Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981827547Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199822024-02-06T22:24:10.837Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981841645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199832024-02-06T22:24:10.837Z{"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981861572Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199842024-02-06T22:24:10.837Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.98187533Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199852024-02-06T22:24:10.837Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981896357Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199862024-02-06T22:24:10.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981910406Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199872024-02-06T22:24:10.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981930793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199882024-02-06T22:24:10.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981944421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199892024-02-06T22:24:10.837Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981965108Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199902024-02-06T22:24:10.837Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981977866Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199912024-02-06T22:24:10.838Z{"msg":"b029f422-97f4-4220-a420-4ecef717e1fa is now active with session: d63f7496-63dd-4801-8be0-7ad9a776d240","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.981990975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199922024-02-06T22:24:10.838Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.982010482Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874,"session_id":"d63f7496-63dd-4801-8be0-7ad9a776d240"}
199932024-02-06T22:24:10.838Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:09.98202352Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874}
199942024-02-06T22:24:10.838ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199952024-02-06T22:24:10.838ZLoading write count information from file "/tmp/test_repair_verify.data"
199962024-02-06T22:24:10.838ZRun Repair workload
199972024-02-06T22:24:10.839Z01/30 Write block 547 len 28672 data: 12 12 12 11 13 15 16
199982024-02-06T22:24:10.839Z02/30 Read block 466 len 40960
199992024-02-06T22:24:10.839Z03/30 Write block 156 len 36864 data: 16 19 15 18 16 17 20 17 16
200002024-02-06T22:24:10.839Z04/30 Read block 146 len 40960
200012024-02-06T22:24:10.839Z05/30 Write block 101 len 12288 data: 10 10 8
200022024-02-06T22:24:10.839Z06/30 Read block 592 len 28672
200032024-02-06T22:24:10.857Z07/30 Read block 525 len 16384
200042024-02-06T22:24:10.857Z08/30 Flush
200052024-02-06T22:24:10.857Z09/30 Write block 270 len 40960 data: 15 14 15 11 9 12 12 13 18 18
200062024-02-06T22:24:10.857Z10/30 Write block 56 len 32768 data: 26 20 18 17 18 14 11 10
200072024-02-06T22:24:10.857Z11/30 Read block 167 len 36864
200082024-02-06T22:24:10.857Z12/30 Read block 483 len 20480
200092024-02-06T22:24:10.857Z13/30 Read block 341 len 20480
200102024-02-06T22:24:10.857Z14/30 Read block 541 len 12288
200112024-02-06T22:24:10.858Z15/30 Write block 422 len 32768 data: 15 13 14 13 14 12 14 12
200122024-02-06T22:24:10.858Z16/30 Read block 320 len 32768
200132024-02-06T22:24:10.858Z17/30 Read block 521 len 20480
200142024-02-06T22:24:10.858Z18/30 Read block 245 len 20480
200152024-02-06T22:24:10.858Z19/30 Write block 79 len 40960 data: 8 9 10 13 13 12 13 14 16 14
200162024-02-06T22:24:10.858Z20/30 Read block 541 len 8192
200172024-02-06T22:24:10.858Z21/30 Write block 279 len 8192 data: 19 19
200182024-02-06T22:24:10.858Z22/30 Write block 0 len 28672 data: 3 6 8 11 10 9 8
200192024-02-06T22:24:10.858Z23/30 Read block 167 len 28672
200202024-02-06T22:24:10.858Z24/30 Read block 561 len 28672
200212024-02-06T22:24:10.859Z25/30 Read block 13 len 24576
200222024-02-06T22:24:10.859Z26/30 Read block 529 len 28672
200232024-02-06T22:24:10.859Z27/30 Read block 384 len 4096
200242024-02-06T22:24:10.859Z28/30 Write block 275 len 24576 data: 13 13 14 19 20 20
200252024-02-06T22:24:10.859Z29/30 Write block 44 len 16384 data: 12 13 13 14
200262024-02-06T22:24:10.859Z30/30 Write block 561 len 20480 data: 12 13 11 13 14
200272024-02-06T22:24:10.859Z----------------------------------------------------------------
200282024-02-06T22:24:10.859Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
200292024-02-06T22:24:10.859ZStates: Active Active Active
200302024-02-06T22:24:10.859ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200312024-02-06T22:24:10.859Z 1 Acked 1000 Write 7 Done Done Done false
200322024-02-06T22:24:10.859Z 2 Acked 1001 Read 10 Done Done Done false
200332024-02-06T22:24:10.859Z 3 Acked 1002 Write 9 Done Done Done false
200342024-02-06T22:24:10.859Z 4 Acked 1003 Read 10 Done Done Done false
200352024-02-06T22:24:10.859Z 5 Acked 1004 Write 3 Sent Done Done false
200362024-02-06T22:24:10.860Z 6 Acked 1005 Read 7 Sent Done Done false
200372024-02-06T22:24:10.860Z 7 Acked 1006 Read 4 Sent Done Done false
200382024-02-06T22:24:10.860Z 8 Acked 1007 Flush 0 Sent Done Done false
200392024-02-06T22:24:10.860Z 9 Acked 1008 Write 10 Sent Done Done false
200402024-02-06T22:24:10.860Z 10 Acked 1009 Write 8 Sent Done Done false
200412024-02-06T22:24:10.860Z 11 Acked 1010 Read 9 Sent Done Done false
200422024-02-06T22:24:10.860Z 12 Acked 1011 Read 5 Sent Done Done false
200432024-02-06T22:24:10.860Z 13 Acked 1012 Read 5 Sent Done Done false
200442024-02-06T22:24:10.860Z 14 Acked 1013 Read 3 Sent Done Done false
200452024-02-06T22:24:10.860Z 15 Acked 1014 Write 8 Sent Done Done false
200462024-02-06T22:24:10.860Z 16 Acked 1015 Read 8 Sent Done Done false
200472024-02-06T22:24:10.860Z 17 Acked 1016 Read 5 Sent Done Done false
200482024-02-06T22:24:10.860Z 18 Acked 1017 Read 5 Sent Done Done false
200492024-02-06T22:24:10.860Z 19 Acked 1018 Write 10 Sent Done Done false
200502024-02-06T22:24:10.860Z 20 Acked 1019 Read 2 Sent Done Done false
200512024-02-06T22:24:10.860Z 21 Acked 1020 Write 2 Sent Done Done false
200522024-02-06T22:24:10.860Z 22 Acked 1021 Write 7 Sent Done Done false
200532024-02-06T22:24:10.860Z 23 Acked 1022 Read 7 Sent Done Done false
200542024-02-06T22:24:10.860Z 24 Acked 1023 Read 7 Sent Done Done false
200552024-02-06T22:24:10.860Z 25 Acked 1024 Read 6 Sent Done Done false
200562024-02-06T22:24:10.860Z 26 Acked 1025 Read 7 Sent Done Done false
200572024-02-06T22:24:10.861Z 27 Acked 1026 Read 1 Sent Done Done false
200582024-02-06T22:24:10.861Z 28 Acked 1027 Write 6 Sent Done Sent false
200592024-02-06T22:24:10.861Z 29 Acked 1028 Write 4 Sent Done Sent false
200602024-02-06T22:24:10.861Z 30 Acked 1029 Write 5 Sent Sent Sent false
200612024-02-06T22:24:10.861Z STATES DS:0 DS:1 DS:2 TOTAL
200622024-02-06T22:24:10.861Z New 0 0 0 0
200632024-02-06T22:24:10.861Z Sent 26 1 3 30
200642024-02-06T22:24:10.861Z Done 4 29 27 60
200652024-02-06T22:24:10.861Z Skipped 0 0 0 0
200662024-02-06T22:24:10.861Z Error 0 0 0 0
200672024-02-06T22:24:10.861ZLast Flush: 0 1007 1007
200682024-02-06T22:24:10.861ZDownstairs last five completed:
200692024-02-06T22:24:10.861ZUpstairs last five completed: 30 29 28 27 26
200702024-02-06T22:24:10.861Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:10.013053987Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1874}
200712024-02-06T22:24:10.861ZWrote out file "/tmp/test_repair_verify.data"
200722024-02-06T22:24:10.869Z
200732024-02-06T22:24:10.870ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200742024-02-06T22:24:10.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200752024-02-06T22:24:10.876Z 0 000-019 181 181 181 326 326 326 F T T <---
200762024-02-06T22:24:10.876Z 1 020-039 181 181 181 326 326 326 F F F
200772024-02-06T22:24:10.876Z 2 040-059 182 182 182 327 327 327 F T T <---
200782024-02-06T22:24:10.876Z 3 060-079 172 172 172 312 312 312 F T T <---
200792024-02-06T22:24:10.876Z 4 080-099 182 182 182 327 327 327 F T T <---
200802024-02-06T22:24:10.876Z 5 100-119 182 184 184 327 329 329 F F F <---
200812024-02-06T22:24:10.876Z 6 120-139 179 179 179 324 324 324 F F F
200822024-02-06T22:24:10.876Z 7 140-159 179 184 184 324 329 329 T F F <---
200832024-02-06T22:24:10.876Z 8 160-179 181 184 184 326 329 329 T F F <---
200842024-02-06T22:24:10.876Z 9 180-199 181 181 181 326 326 326 F F F
200852024-02-06T22:24:10.876Z 10 200-219 176 176 176 320 320 320 F F F
200862024-02-06T22:24:10.876Z 11 220-239 182 182 182 327 327 327 F F F
200872024-02-06T22:24:10.876Z 12 240-259 174 174 174 317 317 317 F F F
200882024-02-06T22:24:10.876Z 13 260-279 181 181 181 326 326 326 F T T <---
200892024-02-06T22:24:10.876Z 14 280-299 181 181 181 326 326 326 F T T <---
200902024-02-06T22:24:10.876Z 15 300-319 183 183 183 328 328 328 F F F
200912024-02-06T22:24:10.876Z 16 320-339 179 179 179 324 324 324 F F F
200922024-02-06T22:24:10.877Z 17 340-359 177 177 177 322 322 322 F F F
200932024-02-06T22:24:10.877Z 18 360-379 182 182 182 327 327 327 F F F
200942024-02-06T22:24:10.877Z 19 380-399 182 182 182 327 327 327 F F F
200952024-02-06T22:24:10.877Z 20 400-419 182 182 182 327 327 327 F F F
200962024-02-06T22:24:10.877Z 21 420-439 182 182 182 327 327 327 F T T <---
200972024-02-06T22:24:10.877Z 22 440-459 181 181 181 326 326 326 F F F
200982024-02-06T22:24:10.877Z 23 460-479 178 178 178 323 323 323 F F F
200992024-02-06T22:24:10.877Z 24 480-499 179 179 179 324 324 324 F F F
201002024-02-06T22:24:10.877Z 25 500-519 183 183 183 328 328 328 F F F
201012024-02-06T22:24:10.877Z 26 520-539 182 182 182 327 327 327 F F F
201022024-02-06T22:24:10.877Z 27 540-559 179 184 184 324 329 329 T F F <---
201032024-02-06T22:24:10.877Z 28 560-579 181 181 181 326 326 326 F T T <---
201042024-02-06T22:24:10.877Z 29 580-599 183 183 183 328 328 328 F F F
201052024-02-06T22:24:10.877ZMax gen: 184, Max flush: 329
201062024-02-06T22:24:10.877ZError: Difference in extent metadata found!
201072024-02-06T22:24:10.877ZA Difference in extent metadata is expected here
201082024-02-06T22:24:10.877ZOn loop 92
201092024-02-06T22:24:10.877Z
201102024-02-06T22:24:10.877ZVerifying data now
201112024-02-06T22:24:16.478ZNo change after verify
201122024-02-06T22:24:16.478ZLoop: 92 Downstairs dump after verify (and repair):
201132024-02-06T22:24:16.492ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201142024-02-06T22:24:16.492Z 0 000-019 185 185 185 330 330 330 F F F
201152024-02-06T22:24:16.492Z 1 020-039 181 181 181 326 326 326 F F F
201162024-02-06T22:24:16.492Z 2 040-059 185 185 185 330 330 330 F F F
201172024-02-06T22:24:16.492Z 3 060-079 185 185 185 330 330 330 F F F
201182024-02-06T22:24:16.492Z 4 080-099 185 185 185 330 330 330 F F F
201192024-02-06T22:24:16.492Z 5 100-119 184 184 184 329 329 329 F F F
201202024-02-06T22:24:16.492Z 6 120-139 179 179 179 324 324 324 F F F
201212024-02-06T22:24:16.492Z 7 140-159 184 184 184 329 329 329 F F F
201222024-02-06T22:24:16.492Z 8 160-179 184 184 184 329 329 329 F F F
201232024-02-06T22:24:16.492Z 9 180-199 181 181 181 326 326 326 F F F
201242024-02-06T22:24:16.492Z 10 200-219 176 176 176 320 320 320 F F F
201252024-02-06T22:24:16.492Z 11 220-239 182 182 182 327 327 327 F F F
201262024-02-06T22:24:16.492Z 12 240-259 174 174 174 317 317 317 F F F
201272024-02-06T22:24:16.498Z 13 260-279 185 185 185 330 330 330 F F F
201282024-02-06T22:24:16.498Z 14 280-299 185 185 185 330 330 330 F F F
201292024-02-06T22:24:16.498Z 15 300-319 183 183 183 328 328 328 F F F
201302024-02-06T22:24:16.498Z 16 320-339 179 179 179 324 324 324 F F F
201312024-02-06T22:24:16.498Z 17 340-359 177 177 177 322 322 322 F F F
201322024-02-06T22:24:16.498Z 18 360-379 182 182 182 327 327 327 F F F
201332024-02-06T22:24:16.498Z 19 380-399 182 182 182 327 327 327 F F F
201342024-02-06T22:24:16.498Z 20 400-419 182 182 182 327 327 327 F F F
201352024-02-06T22:24:16.498Z 21 420-439 185 185 185 330 330 330 F F F
201362024-02-06T22:24:16.498Z 22 440-459 181 181 181 326 326 326 F F F
201372024-02-06T22:24:16.499Z 23 460-479 178 178 178 323 323 323 F F F
201382024-02-06T22:24:16.499Z 24 480-499 179 179 179 324 324 324 F F F
201392024-02-06T22:24:16.499Z 25 500-519 183 183 183 328 328 328 F F F
201402024-02-06T22:24:16.499Z 26 520-539 182 182 182 327 327 327 F F F
201412024-02-06T22:24:16.499Z 27 540-559 184 184 184 329 329 329 F F F
201422024-02-06T22:24:16.499Z 28 560-579 185 185 185 330 330 330 F F F
201432024-02-06T22:24:16.499Z 29 580-599 183 183 183 328 328 328 F F F
201442024-02-06T22:24:16.499ZMax gen: 185, Max flush: 330
201452024-02-06T22:24:16.499Z
201462024-02-06T22:24:16.499ZBegin loop 93
201472024-02-06T22:24:16.499ZDownstairs to restart: 0
201482024-02-06T22:24:16.513Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.682381623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882}
201492024-02-06T22:24:16.513Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.682633178Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882}
201502024-02-06T22:24:16.518Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.682669484Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882}
201512024-02-06T22:24:16.518Z{"msg":"Crucible ca0638f1-15fd-4000-add1-399a1a948d79 has session id: 7b6ad7f3-45ff-4388-b56b-a75359c47788","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.68269431Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201522024-02-06T22:24:16.518Z{"msg":"Upstairs opts: Upstairs UUID: ca0638f1-15fd-4000-add1-399a1a948d79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.682747383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201532024-02-06T22:24:16.518Z{"msg":"Crucible stats registered with UUID: ca0638f1-15fd-4000-add1-399a1a948d79","v":0,"name":"crucible","level":30Crucible runtime is spawned
201542024-02-06T22:24:16.518Z,"time":"2024-02-06T22:24:15.682783948Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201552024-02-06T22:24:16.518Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.682867207Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882}
201562024-02-06T22:24:16.518Z{"msg":"ca0638f1-15fd-4000-add1-399a1a948d79 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683044163Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201572024-02-06T22:24:16.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.68306645Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201582024-02-06T22:24:16.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683084997Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201592024-02-06T22:24:16.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683122642Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201602024-02-06T22:24:16.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.6831403Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201612024-02-06T22:24:16.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683160347Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201622024-02-06T22:24:16.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683184554Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201632024-02-06T22:24:16.519Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683206371Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201642024-02-06T22:24:16.519Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683233137Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201652024-02-06T22:24:16.519Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683281221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201662024-02-06T22:24:16.519Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:15.683373429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201672024-02-06T22:24:16.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683422032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201682024-02-06T22:24:16.519Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:15.683451208Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201692024-02-06T22:24:16.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683493852Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201702024-02-06T22:24:16.519Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:15.683527328Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201712024-02-06T22:24:16.519Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683569202Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201722024-02-06T22:24:16.519Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683592139Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201732024-02-06T22:24:16.519Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.683637553Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201742024-02-06T22:24:16.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.684114958Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201752024-02-06T22:24:16.519Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.684163152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201762024-02-06T22:24:16.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.684206356Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201772024-02-06T22:24:16.519Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.684235452Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201782024-02-06T22:24:16.519Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.684739314Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201792024-02-06T22:24:16.519Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.684796396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201802024-02-06T22:24:16.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.685280011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201812024-02-06T22:24:16.519Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.685318596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201822024-02-06T22:24:16.519Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.685355821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201832024-02-06T22:24:16.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.685380197Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201842024-02-06T22:24:16.519Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.685411383Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201852024-02-06T22:24:16.519Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:15.68543622Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201862024-02-06T22:24:17.516Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.685079876Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201872024-02-06T22:24:17.517Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.685201239Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201882024-02-06T22:24:17.522Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.686131364Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201892024-02-06T22:24:17.522Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.686168079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201902024-02-06T22:24:17.522Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.686626767Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201912024-02-06T22:24:17.522Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687154666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201922024-02-06T22:24:17.522Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687192961Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201932024-02-06T22:24:17.522Z{"msg":"[0]R flush_numbers[0..12]: [330, 326, 330, 330, 330, 329, 324, 329, 329, 326, 320, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687212938Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201942024-02-06T22:24:17.522Z{"msg":"[0]R generation[0..12]: [185, 181, 185, 185, 185, 184, 179, 184, 184, 181, 176, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687232415Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201952024-02-06T22:24:17.522Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687259712Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201962024-02-06T22:24:17.522Z{"msg":"[1]R flush_numbers[0..12]: [330, 326, 330, 330, 330, 329, 324, 329, 329, 326, 320, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687278109Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201972024-02-06T22:24:17.522Z{"msg":"[1]R generation[0..12]: [185, 181, 185, 185, 185, 184, 179, 184, 184, 181, 176, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687305775Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201982024-02-06T22:24:17.522Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687325283Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
201992024-02-06T22:24:17.522Z{"msg":"[2]R flush_numbers[0..12]: [330, 326, 330, 330, 330, 329, 324, 329, 329, 326, 320, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.6873429Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202002024-02-06T22:24:17.522Z{"msg":"[2]R generation[0..12]: [185, 181, 185, 185, 185, 184, 179, 184, 184, 181, 176, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687375456Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202012024-02-06T22:24:17.522Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687398283Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202022024-02-06T22:24:17.522Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687432328Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
202032024-02-06T22:24:17.522Z7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202042024-02-06T22:24:17.522Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687521116Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202052024-02-06T22:24:17.522Z{"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687545213Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202062024-02-06T22:24:17.522Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687588107Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202072024-02-06T22:24:17.522Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687609784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202082024-02-06T22:24:17.522Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.68764069Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202092024-02-06T22:24:17.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687671156Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202102024-02-06T22:24:17.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687692793Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202112024-02-06T22:24:17.523Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687724669Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202122024-02-06T22:24:17.523Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687748075Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202132024-02-06T22:24:17.523Z{"msg":"ca0638f1-15fd-4000-add1-399a1a948d79 is now active with session: 7b6ad7f3-45ff-4388-b56b-a75359c47788","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687781621Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202142024-02-06T22:24:17.523Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687805578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882,"session_id":"7b6ad7f3-45ff-4388-b56b-a75359c47788"}
202152024-02-06T22:24:17.523ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202162024-02-06T22:24:17.523Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:16.687833914Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882}
202172024-02-06T22:24:17.523ZLoading write count information from file "/tmp/test_repair_verify.data"
202182024-02-06T22:24:17.523ZRun Repair workload
202192024-02-06T22:24:17.523Z01/30 Write block 107 len 12288 data: 16 19 14
202202024-02-06T22:24:17.526Z02/30 Write block 277 len 24576 data: 15 20 21 21 18 16
202212024-02-06T22:24:17.527Z03/30 Write block 127 len 36864 data: 18 19 16 18 16 14 16 14 13
202222024-02-06T22:24:17.527Z04/30 Read block 64 len 20480
202232024-02-06T22:24:17.527Z05/30 Write block 327 len 12288 data: 9 7 8
202242024-02-06T22:24:17.527Z06/30 Read block 569 len 36864
202252024-02-06T22:24:17.527Z07/30 Read block 45 len 36864
202262024-02-06T22:24:17.531Z08/30 Read block 434 len 20480
202272024-02-06T22:24:17.531Z09/30 Read block 566 len 36864
202282024-02-06T22:24:17.531Z10/30 Read block 223 len 16384
202292024-02-06T22:24:17.531Z11/30 Write block 317 len 24576 data: 19 18 15 13 15 17
202302024-02-06T22:24:17.534Z12/30 Write block 555 len 40960 data: 15 15 13 14 11 11 13 14 12 14
202312024-02-06T22:24:17.534Z13/30 Read block 384 len 16384
202322024-02-06T22:24:17.534Z14/30 Read block 539 len 28672
202332024-02-06T22:24:17.534Z15/30 Write block 259 len 16384 data: 14 15 20 22
202342024-02-06T22:24:17.534Z16/30 Read block 590 len 36864
202352024-02-06T22:24:17.541Z17/30 Read block 574 len 12288
202362024-02-06T22:24:17.541Z18/30 Write block 477 len 40960 data: 13 13 14 12 10 12 13 16 14 15
202372024-02-06T22:24:17.544Z19/30 Write block 208 len 32768 data: 10 15 14 12 13 16 20 17
202382024-02-06T22:24:17.544Z20/30 Write block 501 len 16384 data: 12 13 17 15
202392024-02-06T22:24:17.544Z21/30 Write block 90 len 20480 data: 12 10 5 6 8
202402024-02-06T22:24:17.544Z22/30 Read block 285 len 16384
202412024-02-06T22:24:17.544Z23/30 Read block 244 len 36864
202422024-02-06T22:24:17.544Z24/30 Read block 510 len 24576
202432024-02-06T22:24:17.544Z25/30 Write block 303 len 4096 data: 17
202442024-02-06T22:24:17.544Z26/30 Write block 245 len 8192 data: 10 11
202452024-02-06T22:24:17.544Z27/30 Read block 171 len 8192
202462024-02-06T22:24:17.548Z28/30 Write block 444 len 20480 data: 10 11 11 9 11
202472024-02-06T22:24:17.548Z29/30 Read block 326 len 24576
202482024-02-06T22:24:17.548Z30/30 Write block 527 len 20480 data: 12 12 17 19 20
202492024-02-06T22:24:17.548Z----------------------------------------------------------------
202502024-02-06T22:24:17.548Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
202512024-02-06T22:24:17.548ZStates: Active Active Active
202522024-02-06T22:24:17.548ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202532024-02-06T22:24:17.548Z 1 Acked 1000 Write 3 Done Done Done false
202542024-02-06T22:24:17.548Z 2 Acked 1001 Write 6 Done Done Done false
202552024-02-06T22:24:17.548Z 3 Acked 1002 Write 9 Done Done Done false
202562024-02-06T22:24:17.548Z 4 Acked 1003 Read 5 Done Done Done false
202572024-02-06T22:24:17.548Z 5 Acked 1004 Write 3 Done Done Done false
202582024-02-06T22:24:17.548Z 6 Acked 1005 Read 9 Done Done Done false
202592024-02-06T22:24:17.548Z 7 Acked 1006 Read 9 Sent Done Done false
202602024-02-06T22:24:17.548Z 8 Acked 1007 Read 5 Sent Done Done false
202612024-02-06T22:24:17.548Z 9 Acked 1008 Read 9 Sent Done Done false
202622024-02-06T22:24:17.548Z 10 Acked 1009 Read 4 Sent Done Done false
202632024-02-06T22:24:17.548Z 11 Acked 1010 Write 6 Sent Done Done false
202642024-02-06T22:24:17.549Z 12 Acked 1011 Write 10 Sent Done Done false
202652024-02-06T22:24:17.549Z 13 Acked 1012 Read 4 Sent Done Done false
202662024-02-06T22:24:17.549Z 14 Acked 1013 Read 7 Sent Done Done false
202672024-02-06T22:24:17.549Z 15 Acked 1014 Write 4 Sent Done Done false
202682024-02-06T22:24:17.549Z 16 Acked 1015 Read 9 Sent Done Done false
202692024-02-06T22:24:17.549Z 17 Acked 1016 Read 3 Sent Done Done false
202702024-02-06T22:24:17.549Z 18 Acked 1017 Write 10 Sent Done Done false
202712024-02-06T22:24:17.549Z 19 Acked 1018 Write 8 Sent Done Done false
202722024-02-06T22:24:17.549Z 20 Acked 1019 Write 4 Sent Done Done false
202732024-02-06T22:24:17.549Z 21 Acked 1020 Write 5 Sent Done Done false
202742024-02-06T22:24:17.549Z 22 Acked 1021 Read 4 Sent Done Done false
202752024-02-06T22:24:17.549Z 23 Acked 1022 Read 9 Sent Done Done false
202762024-02-06T22:24:17.549Z 24 Acked 1023 Read 6 Sent Done Done false
202772024-02-06T22:24:17.549Z 25 Acked 1024 Write 1 Sent Done Done false
202782024-02-06T22:24:17.549Z 26 Acked 1025 Write 2 Sent Done Done false
202792024-02-06T22:24:17.549Z 27 Acked 1026 Read 2 Sent Done Done false
202802024-02-06T22:24:17.549Z 28 Acked 1027 Write 5 Sent Done Done false
202812024-02-06T22:24:17.549Z 29 Acked 1028 Read 6 Sent Done Done false
202822024-02-06T22:24:17.549Z 30 Acked 1029 Write 5 Sent Sent Sent false
202832024-02-06T22:24:17.549Z STATES DS:0 DS:1 DS:2 TOTAL
202842024-02-06T22:24:17.549Z New 0 0 0 0
202852024-02-06T22:24:17.549Z Sent 24 1 1 26
202862024-02-06T22:24:17.549Z Done 6 29 29 64
202872024-02-06T22:24:17.549Z Skipped 0 0 0 0
202882024-02-06T22:24:17.549Z Error 0 0 0 0
202892024-02-06T22:24:17.550ZLast Flush: 0 0 0
202902024-02-06T22:24:17.550ZDownstairs last five completed:
202912024-02-06T22:24:17.550ZUpstairs last five completed: 30 29 28 27 26
202922024-02-06T22:24:17.550Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:16.715860857Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1882}
202932024-02-06T22:24:17.550ZWrote out file "/tmp/test_repair_verify.data"
202942024-02-06T22:24:17.553Z
202952024-02-06T22:24:17.556ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202962024-02-06T22:24:17.576ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202972024-02-06T22:24:17.577Z 0 000-019 185 185 185 330 330 330 F F F
202982024-02-06T22:24:17.577Z 1 020-039 181 181 181 326 326 326 F F F
202992024-02-06T22:24:17.577Z 2 040-059 185 185 185 330 330 330 F F F
203002024-02-06T22:24:17.577Z 3 060-079 185 185 185 330 330 330 F F F
203012024-02-06T22:24:17.577Z 4 080-099 185 185 185 330 330 330 F T T <---
203022024-02-06T22:24:17.577Z 5 100-119 184 184 184 329 329 329 T T T
203032024-02-06T22:24:17.577Z 6 120-139 179 179 179 324 324 324 T T T
203042024-02-06T22:24:17.577Z 7 140-159 184 184 184 329 329 329 F F F
203052024-02-06T22:24:17.577Z 8 160-179 184 184 184 329 329 329 F F F
203062024-02-06T22:24:17.577Z 9 180-199 181 181 181 326 326 326 F F F
203072024-02-06T22:24:17.577Z 10 200-219 176 176 176 320 320 320 F T T <---
203082024-02-06T22:24:17.580Z 11 220-239 182 182 182 327 327 327 F F F
203092024-02-06T22:24:17.580Z 12 240-259 174 174 174 317 317 317 F T T <---
203102024-02-06T22:24:17.580Z 13 260-279 185 185 185 330 330 330 T T T
203112024-02-06T22:24:17.581Z 14 280-299 185 185 185 330 330 330 T T T
203122024-02-06T22:24:17.581Z 15 300-319 183 183 183 328 328 328 F T T <---
203132024-02-06T22:24:17.581Z 16 320-339 179 179 179 324 324 324 T T T
203142024-02-06T22:24:17.581Z 17 340-359 177 177 177 322 322 322 F F F
203152024-02-06T22:24:17.581Z 18 360-379 182 182 182 327 327 327 F F F
203162024-02-06T22:24:17.581Z 19 380-399 182 182 182 327 327 327 F F F
203172024-02-06T22:24:17.581Z 20 400-419 182 182 182 327 327 327 F F F
203182024-02-06T22:24:17.581Z 21 420-439 185 185 185 330 330 330 F F F
203192024-02-06T22:24:17.581Z 22 440-459 181 181 181 326 326 326 F T T <---
203202024-02-06T22:24:17.581Z 23 460-479 178 178 178 323 323 323 F T T <---
203212024-02-06T22:24:17.581Z 24 480-499 179 179 179 324 324 324 F T T <---
203222024-02-06T22:24:17.581Z 25 500-519 183 183 183 328 328 328 F T T <---
203232024-02-06T22:24:17.581Z 26 520-539 182 182 182 327 327 327 F T T <---
203242024-02-06T22:24:17.581Z 27 540-559 184 184 184 329 329 329 F T T <---
203252024-02-06T22:24:17.581Z 28 560-579 185 185 185 330 330 330 F T T <---
203262024-02-06T22:24:17.581Z 29 580-599 183 183 183 328 328 328 F F F
203272024-02-06T22:24:17.581ZMax gen: 185, Max flush: 330
203282024-02-06T22:24:17.581ZError: Difference in extent metadata found!
203292024-02-06T22:24:17.581ZA Difference in extent metadata is expected here
203302024-02-06T22:24:17.581ZOn loop 93
203312024-02-06T22:24:17.581Z
203322024-02-06T22:24:17.585ZVerifying data now
203332024-02-06T22:24:23.412ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
203342024-02-06T22:24:23.415Z263,265c263,264
203352024-02-06T22:24:23.415Z< 14,
203362024-02-06T22:24:23.415Z< 19,
203372024-02-06T22:24:23.416Z< 21,
203382024-02-06T22:24:23.416Z---
203392024-02-06T22:24:23.416Z> 15,
203402024-02-06T22:24:23.416Z> 20,
203412024-02-06T22:24:23.416Z266a266
203422024-02-06T22:24:23.416Z> 22,
203432024-02-06T22:24:23.416Z323,325d322
203442024-02-06T22:24:23.416Z< 12,
203452024-02-06T22:24:23.416Z< 14,
203462024-02-06T22:24:23.416Z< 16,
203472024-02-06T22:24:23.416Z326a324,326
203482024-02-06T22:24:23.416Z> 15,
203492024-02-06T22:24:23.416Z> 17,
203502024-02-06T22:24:23.416Z> 13,
203512024-02-06T22:24:23.416Zdiff found after verify
203522024-02-06T22:24:23.419ZLoop: 93 Downstairs dump after verify (and repair):
203532024-02-06T22:24:23.434ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203542024-02-06T22:24:23.434Z 0 000-019 185 185 185 330 330 330 F F F
203552024-02-06T22:24:23.434Z 1 020-039 181 181 181 326 326 326 F F F
203562024-02-06T22:24:23.434Z 2 040-059 185 185 185 330 330 330 F F F
203572024-02-06T22:24:23.437Z 3 060-079 185 185 185 330 330 330 F F F
203582024-02-06T22:24:23.437Z 4 080-099 186 186 186 331 331 331 F F F
203592024-02-06T22:24:23.437Z 5 100-119 186 186 186 331 331 331 F F F
203602024-02-06T22:24:23.437Z 6 120-139 186 186 186 331 331 331 F F F
203612024-02-06T22:24:23.437Z 7 140-159 184 184 184 329 329 329 F F F
203622024-02-06T22:24:23.437Z 8 160-179 184 184 184 329 329 329 F F F
203632024-02-06T22:24:23.437Z 9 180-199 181 181 181 326 326 326 F F F
203642024-02-06T22:24:23.437Z 10 200-219 186 186 186 331 331 331 F F F
203652024-02-06T22:24:23.438Z 11 220-239 182 182 182 327 327 327 F F F
203662024-02-06T22:24:23.438Z 12 240-259 186 186 186 331 331 331 F F F
203672024-02-06T22:24:23.438Z 13 260-279 186 186 186 331 331 331 F F F
203682024-02-06T22:24:23.438Z 14 280-299 186 186 186 331 331 331 F F F
203692024-02-06T22:24:23.438Z 15 300-319 186 186 186 331 331 331 F F F
203702024-02-06T22:24:23.438Z 16 320-339 186 186 186 331 331 331 F F F
203712024-02-06T22:24:23.438Z 17 340-359 177 177 177 322 322 322 F F F
203722024-02-06T22:24:23.438Z 18 360-379 182 182 182 327 327 327 F F F
203732024-02-06T22:24:23.438Z 19 380-399 182 182 182 327 327 327 F F F
203742024-02-06T22:24:23.438Z 20 400-419 182 182 182 327 327 327 F F F
203752024-02-06T22:24:23.438Z 21 420-439 185 185 185 330 330 330 F F F
203762024-02-06T22:24:23.438Z 22 440-459 186 186 186 331 331 331 F F F
203772024-02-06T22:24:23.438Z 23 460-479 186 186 186 331 331 331 F F F
203782024-02-06T22:24:23.439Z 24 480-499 186 186 186 331 331 331 F F F
203792024-02-06T22:24:23.439Z 25 500-519 186 186 186 331 331 331 F F F
203802024-02-06T22:24:23.439Z 26 520-539 186 186 186 331 331 331 F F F
203812024-02-06T22:24:23.439Z 27 540-559 186 186 186 331 331 331 F F F
203822024-02-06T22:24:23.439Z 28 560-579 186 186 186 331 331 331 F F F
203832024-02-06T22:24:23.439Z 29 580-599 183 183 183 328 328 328 F F F
203842024-02-06T22:24:23.439ZMax gen: 186, Max flush: 331
203852024-02-06T22:24:23.445Z
203862024-02-06T22:24:23.445ZBegin loop 94
203872024-02-06T22:24:23.446ZDownstairs to restart: 1
203882024-02-06T22:24:23.458Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.624869259Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893}
203892024-02-06T22:24:23.459Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625146861Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal"Crucible runtime is spawned
203902024-02-06T22:24:23.459Z,"pid":1893}
203912024-02-06T22:24:23.473Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625191575Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893}
203922024-02-06T22:24:23.474Z{"msg":"Crucible 01c2e4e0-2cf2-435b-900c-cd835b04e739 has session id: 5d7af43c-9799-47b1-92b3-7d20f44630d5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625226271Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
203932024-02-06T22:24:23.474Z{"msg":"Upstairs opts: Upstairs UUID: 01c2e4e0-2cf2-435b-900c-cd835b04e739, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625265496Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
203942024-02-06T22:24:23.474Z{"msg":"Crucible stats registered with UUID: 01c2e4e0-2cf2-435b-900c-cd835b04e739","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.62530699Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
203952024-02-06T22:24:23.474Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625327777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893}
203962024-02-06T22:24:23.474Z{"msg":"01c2e4e0-2cf2-435b-900c-cd835b04e739 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625355703Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
203972024-02-06T22:24:23.474Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.62538121Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
203982024-02-06T22:24:23.474Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625397488Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
203992024-02-06T22:24:23.474Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625411696Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204002024-02-06T22:24:23.474Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625437792Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204012024-02-06T22:24:23.474Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.62545504Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204022024-02-06T22:24:23.475Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625486306Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204032024-02-06T22:24:23.475Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625511322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204042024-02-06T22:24:23.475Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625563265Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204052024-02-06T22:24:23.475Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.62560128Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204062024-02-06T22:24:23.475Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625646084Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204072024-02-06T22:24:23.476Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625690738Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204082024-02-06T22:24:23.476Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:22.62582414Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204092024-02-06T22:24:23.476Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:22.625906359Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204102024-02-06T22:24:23.476Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:22.625957722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204112024-02-06T22:24:23.476Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.625983179Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204122024-02-06T22:24:23.476Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.626017764Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204132024-02-06T22:24:23.476Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.626094424Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204142024-02-06T22:24:23.476Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.626496609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204152024-02-06T22:24:23.476Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.626542393Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204162024-02-06T22:24:23.476Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.626599965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204172024-02-06T22:24:23.476Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.62663746Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204182024-02-06T22:24:23.477Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.627107607Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204192024-02-06T22:24:23.477Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.627264905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204202024-02-06T22:24:23.477Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.627661052Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204212024-02-06T22:24:23.477Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.62774786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204222024-02-06T22:24:23.477Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.627771597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204232024-02-06T22:24:23.477Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.627803273Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204242024-02-06T22:24:23.477Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.62782362Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204252024-02-06T22:24:23.477Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:22.627852786Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204262024-02-06T22:24:24.465Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.627521519Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204272024-02-06T22:24:24.481Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.627742939Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"io task","client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204282024-02-06T22:24:24.482Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.628439905Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204292024-02-06T22:24:24.482Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.628471351Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204302024-02-06T22:24:24.482Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.628980042Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204312024-02-06T22:24:24.482Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629467316Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204322024-02-06T22:24:24.482Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.62950897Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204332024-02-06T22:24:24.482Z{"msg":"[0]R flush_numbers[0..12]: [330, 326, 330, 330, 331, 331, 331, 329, 329, 326, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629542596Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204342024-02-06T22:24:24.482Z{"msg":"[0]R generation[0..12]: [185, 181, 185, 185, 186, 186, 186, 184, 184, 181, 186, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.62958211Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204352024-02-06T22:24:24.482Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629611426Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204362024-02-06T22:24:24.483Z{"msg":"[1]R flush_numbers[0..12]: [330, 326, 330, 330, 331, 331, 331, 329, 329, 326, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629630684Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204372024-02-06T22:24:24.483Z{"msg":"[1]R generation[0..12]: [185, 181, 185, 185, 186, 186, 186, 184, 184, 181, 186, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629663079Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204382024-02-06T22:24:24.483ZWait for a query_work_queue command to finish before sending IO
204392024-02-06T22:24:24.483Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629690466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204402024-02-06T22:24:24.483Z{"msg":"[2]R flush_numbers[0..12]: [330, 326, 330, 330, 331, 331, 331, 329, 329, 326, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629718012Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204412024-02-06T22:24:24.483Z{"msg":"[2]R generation[0..12]: [185, 181, 185, 185, 186, 186, 186, 184, 184, 181, 186, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.62973614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204422024-02-06T22:24:24.483Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629759486Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204432024-02-06T22:24:24.484Z{"msg":"Max found gen is 187","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629773335Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204442024-02-06T22:24:24.484Z{"msg":"Generation requested: 188 >= found:187","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629793712Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204452024-02-06T22:24:24.484Z{"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.62980722Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204462024-02-06T22:24:24.484Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629827157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"":"downstairs","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204472024-02-06T22:24:24.484Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629841565Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204482024-02-06T22:24:24.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629853904Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"0","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204492024-02-06T22:24:24.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.62988136Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"1","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204502024-02-06T22:24:24.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629896138Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"client":"2","":"downstairs","session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204512024-02-06T22:24:24.484Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629916305Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204522024-02-06T22:24:24.484Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629929263Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204532024-02-06T22:24:24.484Z{"msg":"01c2e4e0-2cf2-435b-900c-cd835b04e739 is now active with session: 5d7af43c-9799-47b1-92b3-7d20f44630d5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629948841Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204542024-02-06T22:24:24.485Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629961269Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893,"session_id":"5d7af43c-9799-47b1-92b3-7d20f44630d5"}
204552024-02-06T22:24:24.485Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:23.629980396Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893}
204562024-02-06T22:24:24.485ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204572024-02-06T22:24:24.485ZLoading write count information from file "/tmp/test_repair_verify.data"
204582024-02-06T22:24:24.485ZRun Repair workload
204592024-02-06T22:24:24.485Z01/30 Write block 63 len 4096 data: 11
204602024-02-06T22:24:24.485Z02/30 Write block 370 len 16384 data: 13 11 9 12
204612024-02-06T22:24:24.485Z03/30 Flush
204622024-02-06T22:24:24.485Z04/30 Write block 548 len 12288 data: 13 13 12
204632024-02-06T22:24:24.486Z05/30 Write block 461 len 32768 data: 12 13 11 12 12 13 17 17
204642024-02-06T22:24:24.486Z06/30 Read block 362 len 32768
204652024-02-06T22:24:24.486Z07/30 Write block 51 len 12288 data: 22 22 26
204662024-02-06T22:24:24.486Z08/30 Read block 22 len 8192
204672024-02-06T22:24:24.492Z09/30 Read block 146 len 28672
204682024-02-06T22:24:24.492Z10/30 Flush
204692024-02-06T22:24:24.492Z11/30 Write block 526 len 32768 data: 12 13 13 18 20 21 17 20
204702024-02-06T22:24:24.492Z12/30 Read block 217 len 12288
204712024-02-06T22:24:24.492Z13/30 Read block 73 len 24576
204722024-02-06T22:24:24.492Z14/30 Write block 365 len 12288 data: 15 11 13
204732024-02-06T22:24:24.493Z15/30 Flush
204742024-02-06T22:24:24.493Z16/30 Write block 234 len 8192 data: 18 19
204752024-02-06T22:24:24.493Z17/30 Write block 510 len 12288 data: 17 15 14
204762024-02-06T22:24:24.493Z18/30 Read block 296 len 20480
204772024-02-06T22:24:24.493Z19/30 Write block 307 len 12288 data: 14 14 12
204782024-02-06T22:24:24.493Z20/30 Read block 372 len 32768
204792024-02-06T22:24:24.494Z21/30 Write block 450 len 8192 data: 11 12
204802024-02-06T22:24:24.494Z22/30 Read block 181 len 20480
204812024-02-06T22:24:24.494Z23/30 Write block 506 len 20480 data: 19 18 16 16 18
204822024-02-06T22:24:24.494Z24/30 Read block 83 len 4096
204832024-02-06T22:24:24.494Z25/30 Write block 450 len 32768 data: 12 13 12 14 19 18 13 12
204842024-02-06T22:24:24.494Z26/30 Write block 241 len 12288 data: 10 10 8
204852024-02-06T22:24:24.494Z27/30 Read block 102 len 28672
204862024-02-06T22:24:24.494Z28/30 Read block 422 len 28672
204872024-02-06T22:24:24.511Z29/30 Read block 444 len 16384
204882024-02-06T22:24:24.511Z30/30 Write block 440 len 20480 data: 13 10 12 11 11
204892024-02-06T22:24:24.512Z----------------------------------------------------------------
204902024-02-06T22:24:24.512Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:27
204912024-02-06T22:24:24.512ZStates: Active Active Active
204922024-02-06T22:24:24.512ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204932024-02-06T22:24:24.512Z 4 Acked 1003 Write 3 Done Done Done false
204942024-02-06T22:24:24.513Z 5 Acked 1004 Write 8 Done Done Done false
204952024-02-06T22:24:24.513Z 6 Acked 1005 Read 8 Done Done Done false
204962024-02-06T22:24:24.513Z 7 Acked 1006 Write 3 Done Done Done false
204972024-02-06T22:24:24.513Z 8 Acked 1007 Read 2 Done Sent Done false
204982024-02-06T22:24:24.513Z 9 Acked 1008 Read 7 Done Sent Done false
204992024-02-06T22:24:24.513Z 10 Acked 1009 Flush 0 Done Sent Done false
205002024-02-06T22:24:24.514Z 11 Acked 1010 Write 8 Done Sent Done false
205012024-02-06T22:24:24.514Z 12 Acked 1011 Read 3 Done Sent Done false
205022024-02-06T22:24:24.514Z 13 Acked 1012 Read 6 Done Sent Done false
205032024-02-06T22:24:24.514Z 14 Acked 1013 Write 3 Done Sent Done false
205042024-02-06T22:24:24.514Z 15 Acked 1014 Flush 0 Done Sent Done false
205052024-02-06T22:24:24.514Z 16 Acked 1015 Write 2 Done Sent Done false
205062024-02-06T22:24:24.515Z 17 Acked 1016 Write 3 Done Sent Done false
205072024-02-06T22:24:24.515Z 18 Acked 1017 Read 5 Done Sent Done false
205082024-02-06T22:24:24.515Z 19 Acked 1018 Write 3 Done Sent Done false
205092024-02-06T22:24:24.515Z 20 Acked 1019 Read 8 Done Sent Done false
205102024-02-06T22:24:24.515Z 21 Acked 1020 Write 2 Done Sent Done false
205112024-02-06T22:24:24.515Z 22 Acked 1021 Read 5 Done Sent Done false
205122024-02-06T22:24:24.515Z 23 Acked 1022 Write 5 Done Sent Done false
205132024-02-06T22:24:24.515Z 24 Acked 1023 Read 1 Done Sent Done false
205142024-02-06T22:24:24.516Z 25 Acked 1024 Write 8 Done Sent Done false
205152024-02-06T22:24:24.516Z 26 Acked 1025 Write 3 Done Sent Done false
205162024-02-06T22:24:24.516Z 27 Acked 1026 Read 7 Done Sent Done false
205172024-02-06T22:24:24.516Z 28 Acked 1027 Read 7 Done Sent Done false
205182024-02-06T22:24:24.516Z 29 Acked 1028 Read 4 Done Sent Done false
205192024-02-06T22:24:24.516Z 30 Acked 1029 Write 5 Sent Sent Sent false
205202024-02-06T22:24:24.516Z STATES DS:0 DS:1 DS:2 TOTAL
205212024-02-06T22:24:24.516Z New 0 0 0 0
205222024-02-06T22:24:24.517Z Sent 1 23 1 25
205232024-02-06T22:24:24.517Z Done 26 4 26 56
205242024-02-06T22:24:24.517Z Skipped 0 0 0 0
205252024-02-06T22:24:24.517Z Error 0 0 0 0
205262024-02-06T22:24:24.517ZLast Flush: 1014 1002 1014
205272024-02-06T22:24:24.517ZDownstairs last five completed: 1002 1001 1000
205282024-02-06T22:24:24.517ZUpstairs last five completed: 30 29 28 27 26
205292024-02-06T22:24:24.517Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:23.659957276Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1893}
205302024-02-06T22:24:24.517ZWrote out file "/tmp/test_repair_verify.data"
205312024-02-06T22:24:24.517Z
205322024-02-06T22:24:24.518ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205332024-02-06T22:24:24.534ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205342024-02-06T22:24:24.534Z 0 000-019 185 185 185 330 330 330 F F F
205352024-02-06T22:24:24.535Z 1 020-039 181 181 181 326 326 326 F F F
205362024-02-06T22:24:24.535Z 2 040-059 188 185 188 333 330 333 F T F <---
205372024-02-06T22:24:24.535Z 3 060-079 188 188 188 332 332 332 F F F
205382024-02-06T22:24:24.535Z 4 080-099 186 186 186 331 331 331 F F F
205392024-02-06T22:24:24.535Z 5 100-119 186 186 186 331 331 331 F F F
205402024-02-06T22:24:24.535Z 6 120-139 186 186 186 331 331 331 F F F
205412024-02-06T22:24:24.535Z 7 140-159 184 184 184 329 329 329 F F F
205422024-02-06T22:24:24.536Z 8 160-179 184 184 184 329 329 329 F F F
205432024-02-06T22:24:24.536Z 9 180-199 181 181 181 326 326 326 F F F
205442024-02-06T22:24:24.536Z 10 200-219 186 186 186 331 331 331 F F F
205452024-02-06T22:24:24.536Z 11 220-239 182 182 182 327 327 327 T F T <---
205462024-02-06T22:24:24.536Z 12 240-259 186 186 186 331 331 331 T F T <---
205472024-02-06T22:24:24.536Z 13 260-279 186 186 186 331 331 331 F F F
205482024-02-06T22:24:24.537Z 14 280-299 186 186 186 331 331 331 F F F
205492024-02-06T22:24:24.537Z 15 300-319 186 186 186 331 331 331 T F T <---
205502024-02-06T22:24:24.537Z 16 320-339 186 186 186 331 331 331 F F F
205512024-02-06T22:24:24.537Z 17 340-359 177 177 177 322 322 322 F F F
205522024-02-06T22:24:24.537Z 18 360-379 188 188 188 334 332 334 F F F <---
205532024-02-06T22:24:24.537Z 19 380-399 182 182 182 327 327 327 F F F
205542024-02-06T22:24:24.537Z 20 400-419 182 182 182 327 327 327 F F F
205552024-02-06T22:24:24.537Z 21 420-439 185 185 185 330 330 330 F F F
205562024-02-06T22:24:24.538Z 22 440-459 186 186 186 331 331 331 T F T <---
205572024-02-06T22:24:24.538Z 23 460-479 188 186 188 333 331 333 F T F <---
205582024-02-06T22:24:24.538Z 24 480-499 186 186 186 331 331 331 F F F
205592024-02-06T22:24:24.538Z 25 500-519 186 186 186 331 331 331 T F T <---
205602024-02-06T22:24:24.538Z 26 520-539 188 186 188 334 331 334 F F F <---
205612024-02-06T22:24:24.538Z 27 540-559 188 186 188 333 331 333 F T F <---
205622024-02-06T22:24:24.538Z 28 560-579 186 186 186 331 331 331 F F F
205632024-02-06T22:24:24.539Z 29 580-599 183 183 183 328 328 328 F F F
205642024-02-06T22:24:24.539ZMax gen: 188, Max flush: 334
205652024-02-06T22:24:24.539ZError: Difference in extent metadata found!
205662024-02-06T22:24:24.539ZA Difference in extent metadata is expected here
205672024-02-06T22:24:24.539ZOn loop 94
205682024-02-06T22:24:24.539Z
205692024-02-06T22:24:24.554ZVerifying data now
205702024-02-06T22:24:30.049ZNo change after verify
205712024-02-06T22:24:30.049ZLoop: 94 Downstairs dump after verify (and repair):
205722024-02-06T22:24:30.065ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205732024-02-06T22:24:30.065Z 0 000-019 185 185 185 330 330 330 F F F
205742024-02-06T22:24:30.065Z 1 020-039 181 181 181 326 326 326 F F F
205752024-02-06T22:24:30.065Z 2 040-059 188 188 188 333 333 333 F F F
205762024-02-06T22:24:30.065Z 3 060-079 188 188 188 332 332 332 F F F
205772024-02-06T22:24:30.065Z 4 080-099 186 186 186 331 331 331 F F F
205782024-02-06T22:24:30.065Z 5 100-119 186 186 186 331 331 331 F F F
205792024-02-06T22:24:30.069Z 6 120-139 186 186 186 331 331 331 F F F
205802024-02-06T22:24:30.069Z 7 140-159 184 184 184 329 329 329 F F F
205812024-02-06T22:24:30.069Z 8 160-179 184 184 184 329 329 329 F F F
205822024-02-06T22:24:30.069Z 9 180-199 181 181 181 326 326 326 F F F
205832024-02-06T22:24:30.069Z 10 200-219 186 186 186 331 331 331 F F F
205842024-02-06T22:24:30.069Z 11 220-239 189 189 189 335 335 335 F F F
205852024-02-06T22:24:30.069Z 12 240-259 189 189 189 335 335 335 F F F
205862024-02-06T22:24:30.069Z 13 260-279 186 186 186 331 331 331 F F F
205872024-02-06T22:24:30.069Z 14 280-299 186 186 186 331 331 331 F F F
205882024-02-06T22:24:30.069Z 15 300-319 189 189 189 335 335 335 F F F
205892024-02-06T22:24:30.069Z 16 320-339 186 186 186 331 331 331 F F F
205902024-02-06T22:24:30.069Z 17 340-359 177 177 177 322 322 322 F F F
205912024-02-06T22:24:30.069Z 18 360-379 188 188 188 334 334 334 F F F
205922024-02-06T22:24:30.069Z 19 380-399 182 182 182 327 327 327 F F F
205932024-02-06T22:24:30.069Z 20 400-419 182 182 182 327 327 327 F F F
205942024-02-06T22:24:30.069Z 21 420-439 185 185 185 330 330 330 F F F
205952024-02-06T22:24:30.069Z 22 440-459 189 189 189 335 335 335 F F F
205962024-02-06T22:24:30.069Z 23 460-479 188 188 188 333 333 333 F F F
205972024-02-06T22:24:30.069Z 24 480-499 186 186 186 331 331 331 F F F
205982024-02-06T22:24:30.069Z 25 500-519 189 189 189 335 335 335 F F F
205992024-02-06T22:24:30.070Z 26 520-539 188 188 188 334 334 334 F F F
206002024-02-06T22:24:30.070Z 27 540-559 188 188 188 333 333 333 F F F
206012024-02-06T22:24:30.070Z 28 560-579 186 186 186 331 331 331 F F F
206022024-02-06T22:24:30.070Z 29 580-599 183 183 183 328 328 328 F F F
206032024-02-06T22:24:30.070ZMax gen: 189, Max flush: 335
206042024-02-06T22:24:30.070Z
206052024-02-06T22:24:30.070ZBegin loop 95
206062024-02-06T22:24:30.070ZDownstairs to restart: 1
206072024-02-06T22:24:30.086Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.25343117Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901}
206082024-02-06T22:24:30.086Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253665959Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901}
206092024-02-06T22:24:30.087Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253704323Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901}
206102024-02-06T22:24:30.092Z{"msg":"Crucible 694d298b-3ec2-454a-b03b-94ba803ade4d has session id: 8837a2dd-afeb-4a49-9fe1-ac9f4be65c25","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.25372929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"Crucible runtime is spawned
206112024-02-06T22:24:30.092Z}
206122024-02-06T22:24:30.092Z{"msg":"Upstairs opts: Upstairs UUID: 694d298b-3ec2-454a-b03b-94ba803ade4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253776473Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206132024-02-06T22:24:30.092Z{"msg":"Crucible stats registered with UUID: 694d298b-3ec2-454a-b03b-94ba803ade4d","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253822217Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206142024-02-06T22:24:30.092Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253856513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901}
206152024-02-06T22:24:30.092Z{"msg":"694d298b-3ec2-454a-b03b-94ba803ade4d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253886189Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206162024-02-06T22:24:30.092Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253911595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206172024-02-06T22:24:30.092Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253945011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206182024-02-06T22:24:30.092Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253963428Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206192024-02-06T22:24:30.092Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.253988295Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206202024-02-06T22:24:30.092Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254004033Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206212024-02-06T22:24:30.092Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254019131Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206222024-02-06T22:24:30.092Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254057346Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206232024-02-06T22:24:30.092Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254079303Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206242024-02-06T22:24:30.092Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254110399Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206252024-02-06T22:24:30.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254259468Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206262024-02-06T22:24:30.092Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.25431898Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206272024-02-06T22:24:30.092Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:29.254418667Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206282024-02-06T22:24:30.092Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:29.2544706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206292024-02-06T22:24:30.093Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:29.254506855Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206302024-02-06T22:24:30.093Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254538401Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206312024-02-06T22:24:30.093Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254571686Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206322024-02-06T22:24:30.093Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.25461463Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206332024-02-06T22:24:30.093Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.254970372Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206342024-02-06T22:24:30.093Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.255016666Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206352024-02-06T22:24:30.093Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.255055391Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206362024-02-06T22:24:30.093Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.255090946Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206372024-02-06T22:24:30.093Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.255520298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206382024-02-06T22:24:30.093Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.255682816Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206392024-02-06T22:24:30.093Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.256076763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206402024-02-06T22:24:30.093Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.256132525Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206412024-02-06T22:24:30.093Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.256163751Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206422024-02-06T22:24:30.093Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.256204105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206432024-02-06T22:24:30.093Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.256223083Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206442024-02-06T22:24:30.093Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:29.256237241Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206452024-02-06T22:24:31.089Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.25600961Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206462024-02-06T22:24:31.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.256093748Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206472024-02-06T22:24:31.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.256925066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206482024-02-06T22:24:31.094Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.25697229Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206492024-02-06T22:24:31.094Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.257449455Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206502024-02-06T22:24:31.094Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258014529Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206512024-02-06T22:24:31.094Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258061493Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206522024-02-06T22:24:31.094Z{"msg":"[0]R flush_numbers[0..12]: [330, 326, 333, 332, 331, 331, 331, 329, 329, 326, 331, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258089279Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206532024-02-06T22:24:31.094Z{"msg":"[0]R generation[0..12]: [185, 181, 188, 188, 186, 186, 186, 184, 184, 181, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258115785Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206542024-02-06T22:24:31.094Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258132483Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206552024-02-06T22:24:31.094Z{"msg":"[1]R flush_numbers[0..12]: [330, 326, 333, 332, 331, 331, 331, 329, 329, 326, 331, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258149421Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206562024-02-06T22:24:31.094Z{"msg":"[1]R generation[0..12]: [185, 181, 188, 188, 186, 186, 186, 184, 184, 181, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258171558Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206572024-02-06T22:24:31.095Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258187205Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206582024-02-06T22:24:31.095Z{"msg":"[2]R flush_numbers[0..12]: [330, 326, 333, 332, 331, 331, 331, 329, 329, 326, 331, 335]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258203083Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206592024-02-06T22:24:31.095Z{"msg":"[2]R generation[0..12]: [185, 181, 188, 188, 186, 186, 186, 184, 184, 181, 186, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258217112Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206602024-02-06T22:24:31.095Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258240068Z","hostname":"Wait for a query_work_queue command to finish before sending IO
206612024-02-06T22:24:31.095Zip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206622024-02-06T22:24:31.095Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258257636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206632024-02-06T22:24:31.095Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258271754Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206642024-02-06T22:24:31.095Z{"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258293051Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206652024-02-06T22:24:31.095Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258306889Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206662024-02-06T22:24:31.095Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258321327Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206672024-02-06T22:24:31.095Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258333486Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206682024-02-06T22:24:31.095Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258355063Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206692024-02-06T22:24:31.095Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258368911Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206702024-02-06T22:24:31.095Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258382919Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206712024-02-06T22:24:31.095Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258402946Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206722024-02-06T22:24:31.095Z{"msg":"694d298b-3ec2-454a-b03b-94ba803ade4d is now active with session: 8837a2dd-afeb-4a49-9fe1-ac9f4be65c25","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258416454Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206732024-02-06T22:24:31.095Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.258430173Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901,"session_id":"8837a2dd-afeb-4a49-9fe1-ac9f4be65c25"}
206742024-02-06T22:24:31.095Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:30.25845142Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901}
206752024-02-06T22:24:31.095ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206762024-02-06T22:24:31.095ZLoading write count information from file "/tmp/test_repair_verify.data"
206772024-02-06T22:24:31.095ZRun Repair workload
206782024-02-06T22:24:31.095Z01/30 Write block 515 len 16384 data: 16 14 16 14
206792024-02-06T22:24:31.099Z02/30 Read block 22 len 20480
206802024-02-06T22:24:31.099Z03/30 Write block 216 len 24576 data: 14 11 12 12 16 17
206812024-02-06T22:24:31.099Z04/30 Read block 337 len 8192
206822024-02-06T22:24:31.099Z05/30 Flush
206832024-02-06T22:24:31.099Z06/30 Write block 200 len 12288 data: 10 13 17
206842024-02-06T22:24:31.103Z07/30 Read block 234 len 40960
206852024-02-06T22:24:31.103Z08/30 Read block 369 len 20480
206862024-02-06T22:24:31.103Z09/30 Write block 21 len 8192 data: 9 11
206872024-02-06T22:24:31.103Z10/30 Write block 141 len 12288 data: 10 13 16
206882024-02-06T22:24:31.103Z11/30 Write block 226 len 4096 data: 18
206892024-02-06T22:24:31.103Z12/30 Read block 522 len 20480
206902024-02-06T22:24:31.107Z13/30 Flush
206912024-02-06T22:24:31.107Z14/30 Write block 111 len 24576 data: 14 13 14 16 18 17
206922024-02-06T22:24:31.107Z15/30 Read block 318 len 12288
206932024-02-06T22:24:31.110Z16/30 Write block 463 len 16384 data: 12 13 13 14
206942024-02-06T22:24:31.110Z17/30 Flush
206952024-02-06T22:24:31.113Z18/30 Write block 75 len 20480 data: 14 11 12 11 9
206962024-02-06T22:24:31.113Z19/30 Write block 374 len 12288 data: 12 11 12
206972024-02-06T22:24:31.113Z20/30 Write block 374 len 20480 data: 13 12 13 14 14
206982024-02-06T22:24:31.116Z21/30 Write block 503 len 36864 data: 18 16 18 20 19 17 17 19 16
206992024-02-06T22:24:31.116Z22/30 Read block 365 len 20480
207002024-02-06T22:24:31.116Z23/30 Write block 197 len 24576 data: 13 11 11 11 14 18
207012024-02-06T22:24:31.119Z24/30 Write block 413 len 28672 data: 14 13 16 16 18 18 18
207022024-02-06T22:24:31.119Z25/30 Flush
207032024-02-06T22:24:31.122Z26/30 Write block 52 len 12288 data: 23 27 30
207042024-02-06T22:24:31.122Z27/30 Write block 486 len 4096 data: 16
207052024-02-06T22:24:31.122Z28/30 Read block 185 len 16384
207062024-02-06T22:24:31.122Z29/30 Write block 532 len 28672 data: 18 21 14 13 12 11 9
207072024-02-06T22:24:31.126Z30/30 Read block 376 len 12288
207082024-02-06T22:24:31.126Z----------------------------------------------------------------
207092024-02-06T22:24:31.126Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
207102024-02-06T22:24:31.126ZStates: Active Active Active
207112024-02-06T22:24:31.126ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207122024-02-06T22:24:31.126Z 1 Acked 1000 Write 4 Done Done Done false
207132024-02-06T22:24:31.126Z 2 Acked 1001 Read 5 Done Done Done false
207142024-02-06T22:24:31.126Z 3 Acked 1002 Write 6 Done Sent Done false
207152024-02-06T22:24:31.126Z 4 Acked 1003 Read 2 Done Sent Done false
207162024-02-06T22:24:31.126Z 5 Acked 1004 Flush 0 Done Sent Done false
207172024-02-06T22:24:31.126Z 6 Acked 1005 Write 3 Done Sent Done false
207182024-02-06T22:24:31.126Z 7 Acked 1006 Read 10 Done Sent Done false
207192024-02-06T22:24:31.126Z 8 Acked 1007 Read 5 Done Sent Done false
207202024-02-06T22:24:31.126Z 9 Acked 1008 Write 2 Done Sent Done false
207212024-02-06T22:24:31.126Z 10 Acked 1009 Write 3 Done Sent Done false
207222024-02-06T22:24:31.126Z 11 Acked 1010 Write 1 Done Sent Done false
207232024-02-06T22:24:31.126Z 12 Acked 1011 Read 5 Done Sent Done false
207242024-02-06T22:24:31.126Z 13 Acked 1012 Flush 0 Done Sent Done false
207252024-02-06T22:24:31.126Z 14 Acked 1013 Write 6 Done Sent Done false
207262024-02-06T22:24:31.126Z 15 Acked 1014 Read 3 Done Sent Done false
207272024-02-06T22:24:31.126Z 16 Acked 1015 Write 4 Done Sent Done false
207282024-02-06T22:24:31.126Z 17 Acked 1016 Flush 0 Done Sent Done false
207292024-02-06T22:24:31.126Z 18 Acked 1017 Write 5 Done Sent Done false
207302024-02-06T22:24:31.127Z 19 Acked 1018 Write 3 Done Sent Done false
207312024-02-06T22:24:31.127Z 20 Acked 1019 Write 5 Done Sent Done false
207322024-02-06T22:24:31.127Z 21 Acked 1020 Write 9 Done Sent Done false
207332024-02-06T22:24:31.127Z 22 Acked 1021 Read 5 Done Sent Done false
207342024-02-06T22:24:31.127Z 23 Acked 1022 Write 6 Done Sent Done false
207352024-02-06T22:24:31.127Z 24 Acked 1023 Write 7 Done Sent Done false
207362024-02-06T22:24:31.127Z 25 Acked 1024 Flush 0 Done Sent Done false
207372024-02-06T22:24:31.127Z 26 Acked 1025 Write 3 Done Sent Done false
207382024-02-06T22:24:31.127Z 27 Acked 1026 Write 1 Done Sent Done false
207392024-02-06T22:24:31.127Z 28 Acked 1027 Read 4 Done Sent Done false
207402024-02-06T22:24:31.127Z 29 Acked 1028 Write 7 Done Sent Done false
207412024-02-06T22:24:31.127Z 30 Acked 1029 Read 3 Done Sent Done false
207422024-02-06T22:24:31.127Z STATES DS:0 DS:1 DS:2 TOTAL
207432024-02-06T22:24:31.127Z New 0 0 0 0
207442024-02-06T22:24:31.127Z Sent 0 28 0 28
207452024-02-06T22:24:31.127Z Done 30 2 30 62
207462024-02-06T22:24:31.127Z Skipped 0 0 0 0
207472024-02-06T22:24:31.127Z Error 0 0 0 0
207482024-02-06T22:24:31.127ZLast Flush: 1024 0 1024
207492024-02-06T22:24:31.127ZDownstairs last five completed:
207502024-02-06T22:24:31.127ZUpstairs last five completed: 30 29 28 27 26
207512024-02-06T22:24:31.127Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:30.291026439Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1901}
207522024-02-06T22:24:31.127ZWrote out file "/tmp/test_repair_verify.data"
207532024-02-06T22:24:31.131Z
207542024-02-06T22:24:31.134ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207552024-02-06T22:24:31.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207562024-02-06T22:24:31.153Z 0 000-019 185 185 185 330 330 330 F F F
207572024-02-06T22:24:31.153Z 1 020-039 190 181 190 337 326 337 F F F <---
207582024-02-06T22:24:31.153Z 2 040-059 188 188 188 333 333 333 T F T <---
207592024-02-06T22:24:31.153Z 3 060-079 190 188 190 339 332 339 F F F <---
207602024-02-06T22:24:31.153Z 4 080-099 186 186 186 331 331 331 F F F
207612024-02-06T22:24:31.153Z 5 100-119 190 186 190 338 331 338 F F F <---
207622024-02-06T22:24:31.153Z 6 120-139 186 186 186 331 331 331 F F F
207632024-02-06T22:24:31.157Z 7 140-159 190 184 190 337 329 337 F F F <---
207642024-02-06T22:24:31.157Z 8 160-179 184 184 184 329 329 329 F F F
207652024-02-06T22:24:31.157Z 9 180-199 190 181 190 339 326 339 F F F <---
207662024-02-06T22:24:31.157Z 10 200-219 190 186 190 339 331 339 F F F <---
207672024-02-06T22:24:31.157Z 11 220-239 190 189 190 337 335 337 F F F <---
207682024-02-06T22:24:31.157Z 12 240-259 189 189 189 335 335 335 F F F
207692024-02-06T22:24:31.157Z 13 260-279 186 186 186 331 331 331 F F F
207702024-02-06T22:24:31.157Z 14 280-299 186 186 186 331 331 331 F F F
207712024-02-06T22:24:31.157Z 15 300-319 189 189 189 335 335 335 F F F
207722024-02-06T22:24:31.157Z 16 320-339 186 186 186 331 331 331 F F F
207732024-02-06T22:24:31.157Z 17 340-359 177 177 177 322 322 322 F F F
207742024-02-06T22:24:31.157Z 18 360-379 190 188 190 339 334 339 F F F <---
207752024-02-06T22:24:31.157Z 19 380-399 182 182 182 327 327 327 F F F
207762024-02-06T22:24:31.157Z 20 400-419 190 182 190 339 327 339 F F F <---
207772024-02-06T22:24:31.158Z 21 420-439 185 185 185 330 330 330 F F F
207782024-02-06T22:24:31.158Z 22 440-459 189 189 189 335 335 335 F F F
207792024-02-06T22:24:31.158Z 23 460-479 190 188 190 338 333 338 F F F <---
207802024-02-06T22:24:31.158Z 24 480-499 186 186 186 331 331 331 T F T <---
207812024-02-06T22:24:31.158Z 25 500-519 190 189 190 339 335 339 F T F <---
207822024-02-06T22:24:31.158Z 26 520-539 188 188 188 334 334 334 T F T <---
207832024-02-06T22:24:31.158Z 27 540-559 188 188 188 333 333 333 F F F
207842024-02-06T22:24:31.158Z 28 560-579 186 186 186 331 331 331 F F F
207852024-02-06T22:24:31.158Z 29 580-599 183 183 183 328 328 328 F F F
207862024-02-06T22:24:31.158ZMax gen: 190, Max flush: 339
207872024-02-06T22:24:31.158ZError: Difference in extent metadata found!
207882024-02-06T22:24:31.158ZA Difference in extent metadata is expected here
207892024-02-06T22:24:31.158ZOn loop 95
207902024-02-06T22:24:31.158Z
207912024-02-06T22:24:31.158ZVerifying data now
207922024-02-06T22:24:35.837ZNo change after verify
207932024-02-06T22:24:35.837ZLoop: 95 Downstairs dump after verify (and repair):
207942024-02-06T22:24:35.853ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207952024-02-06T22:24:35.853Z 0 000-019 185 185 185 330 330 330 F F F
207962024-02-06T22:24:35.853Z 1 020-039 190 190 190 337 337 337 F F F
207972024-02-06T22:24:35.853Z 2 040-059 191 191 191 340 340 340 F F F
207982024-02-06T22:24:35.853Z 3 060-079 190 190 190 339 339 339 F F F
207992024-02-06T22:24:35.853Z 4 080-099 186 186 186 331 331 331 F F F
208002024-02-06T22:24:35.853Z 5 100-119 190 190 190 338 338 338 F F F
208012024-02-06T22:24:35.853Z 6 120-139 186 186 186 331 331 331 F F F
208022024-02-06T22:24:35.853Z 7 140-159 190 190 190 337 337 337 F F F
208032024-02-06T22:24:35.853Z 8 160-179 184 184 184 329 329 329 F F F
208042024-02-06T22:24:35.853Z 9 180-199 190 190 190 339 339 339 F F F
208052024-02-06T22:24:35.853Z 10 200-219 190 190 190 339 339 339 F F F
208062024-02-06T22:24:35.853Z 11 220-239 190 190 190 337 337 337 F F F
208072024-02-06T22:24:35.853Z 12 240-259 189 189 189 335 335 335 F F F
208082024-02-06T22:24:35.853Z 13 260-279 186 186 186 331 331 331 F F F
208092024-02-06T22:24:35.854Z 14 280-299 186 186 186 331 331 331 F F F
208102024-02-06T22:24:35.854Z 15 300-319 189 189 189 335 335 335 F F F
208112024-02-06T22:24:35.854Z 16 320-339 186 186 186 331 331 331 F F F
208122024-02-06T22:24:35.857Z 17 340-359 177 177 177 322 322 322 F F F
208132024-02-06T22:24:35.857Z 18 360-379 190 190 190 339 339 339 F F F
208142024-02-06T22:24:35.857Z 19 380-399 182 182 182 327 327 327 F F F
208152024-02-06T22:24:35.857Z 20 400-419 190 190 190 339 339 339 F F F
208162024-02-06T22:24:35.857Z 21 420-439 185 185 185 330 330 330 F F F
208172024-02-06T22:24:35.857Z 22 440-459 189 189 189 335 335 335 F F F
208182024-02-06T22:24:35.857Z 23 460-479 190 190 190 338 338 338 F F F
208192024-02-06T22:24:35.857Z 24 480-499 191 191 191 340 340 340 F F F
208202024-02-06T22:24:35.857Z 25 500-519 190 190 190 339 339 339 F F F
208212024-02-06T22:24:35.857Z 26 520-539 191 191 191 340 340 340 F F F
208222024-02-06T22:24:35.857Z 27 540-559 188 188 188 333 333 333 F F F
208232024-02-06T22:24:35.857Z 28 560-579 186 186 186 331 331 331 F F F
208242024-02-06T22:24:35.858Z 29 580-599 183 183 183 328 328 328 F F F
208252024-02-06T22:24:35.858ZMax gen: 191, Max flush: 340
208262024-02-06T22:24:35.858Z
208272024-02-06T22:24:35.858ZBegin loop 96
208282024-02-06T22:24:35.858ZDownstairs to restart: 0
208292024-02-06T22:24:35.875Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.041342404Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909}
208302024-02-06T22:24:35.875Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.041621126Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909}
208312024-02-06T22:24:35.880Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.041646733Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909}
208322024-02-06T22:24:35.880Z{"msg":"Crucible c5701e21-f738-4355-aeca-7abe0f3a8976 has session id: 2c469060-693b-4761-a748-ad9dc2fa8482","v":0,"name":"crucible","level":30Crucible runtime is spawned
208332024-02-06T22:24:35.880Z,"time":"2024-02-06T22:24:35.041704795Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208342024-02-06T22:24:35.880Z{"msg":"Upstairs opts: Upstairs UUID: c5701e21-f738-4355-aeca-7abe0f3a8976, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04174136Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208352024-02-06T22:24:35.880Z{"msg":"Crucible stats registered with UUID: c5701e21-f738-4355-aeca-7abe0f3a8976","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.041779585Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208362024-02-06T22:24:35.880Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04181464Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909}
208372024-02-06T22:24:35.880Z{"msg":"c5701e21-f738-4355-aeca-7abe0f3a8976 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.041948692Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208382024-02-06T22:24:35.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042007644Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208392024-02-06T22:24:35.880Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042058367Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208402024-02-06T22:24:35.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042096512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208412024-02-06T22:24:35.880Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042208117Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208422024-02-06T22:24:35.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042312193Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208432024-02-06T22:24:35.880Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042379294Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208442024-02-06T22:24:35.880Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042430717Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208452024-02-06T22:24:35.880Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04247942Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208462024-02-06T22:24:35.880Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042517225Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208472024-02-06T22:24:35.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042563369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208482024-02-06T22:24:35.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.042589375Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208492024-02-06T22:24:35.881Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04262836Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208502024-02-06T22:24:35.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043065931Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208512024-02-06T22:24:35.881Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043115924Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208522024-02-06T22:24:35.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043162088Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208532024-02-06T22:24:35.881Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043191104Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208542024-02-06T22:24:35.881Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043822879Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208552024-02-06T22:24:35.881Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043898628Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208562024-02-06T22:24:35.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043950011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208572024-02-06T22:24:35.881Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.043975548Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208582024-02-06T22:24:35.881Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.044362636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208592024-02-06T22:24:35.881Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.044391582Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208602024-02-06T22:24:35.881Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.0444066Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208612024-02-06T22:24:35.881Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.044473721Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208622024-02-06T22:24:35.881Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.044502177Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208632024-02-06T22:24:35.881Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.044517425Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208642024-02-06T22:24:35.881Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04470009Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208652024-02-06T22:24:35.886Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045173246Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208662024-02-06T22:24:35.886Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04521922Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208672024-02-06T22:24:35.886Z{"msg":"[0]R flush_numbers[0..12]: [330, 337, 340, 339, 331, 338, 331, 337, 329, 339, 339, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045238597Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208682024-02-06T22:24:35.886Z{"msg":"[0]R generation[0..12]: [185, 190, 191, 190, 186, 190, 186, 190, 184, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045254615Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208692024-02-06T22:24:35.886Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045269253Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208702024-02-06T22:24:35.886Z{"msg":"[1]R flush_numbers[0..12]: [330, 337, 340, 339, 331, 338, 331, 337, 329, 339, 339, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04529181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208712024-02-06T22:24:35.886Z{"msg":"[1]R generation[0..12]: [185, 190, 191, 190, 186, 190, 186, 190, 184, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045307298Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208722024-02-06T22:24:35.886Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045340004Z","Wait for a query_work_queue command to finish before sending IO
208732024-02-06T22:24:35.886Zhostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208742024-02-06T22:24:35.886Z{"msg":"[2]R flush_numbers[0..12]: [330, 337, 340, 339, 331, 338, 331, 337, 329, 339, 339, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045380508Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208752024-02-06T22:24:35.886Z{"msg":"[2]R generation[0..12]: [185, 190, 191, 190, 186, 190, 186, 190, 184, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045414004Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208762024-02-06T22:24:35.886Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045431471Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208772024-02-06T22:24:35.886Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045451799Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208782024-02-06T22:24:35.886Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045473456Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208792024-02-06T22:24:35.887Z{"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045509101Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208802024-02-06T22:24:35.887Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045540156Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208812024-02-06T22:24:35.887Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045556844Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208822024-02-06T22:24:35.887Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045585741Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208832024-02-06T22:24:35.887Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045622965Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208842024-02-06T22:24:35.887Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045684787Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208852024-02-06T22:24:35.887Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045721002Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208862024-02-06T22:24:35.887Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.04573615Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208872024-02-06T22:24:35.887Z{"msg":"c5701e21-f738-4355-aeca-7abe0f3a8976 is now active with session: 2c469060-693b-4761-a748-ad9dc2fa8482","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045749888Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208882024-02-06T22:24:35.887Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045770856Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909,"session_id":"2c469060-693b-4761-a748-ad9dc2fa8482"}
208892024-02-06T22:24:35.887Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:35.045784234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909}
208902024-02-06T22:24:35.887ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208912024-02-06T22:24:35.887ZLoading write count information from file "/tmp/test_repair_verify.data"
208922024-02-06T22:24:35.887ZRun Repair workload
208932024-02-06T22:24:35.887Z01/30 Write block 443 len 28672 data: 12 12 12 12 10 12 11
208942024-02-06T22:24:35.887Z02/30 Read block 351 len 36864
208952024-02-06T22:24:35.887Z03/30 Read block 562 len 40960
208962024-02-06T22:24:35.887Z04/30 Read block 455 len 24576
208972024-02-06T22:24:35.890Z05/30 Write block 460 len 32768 data: 13 13 14 13 14 14 15 18
208982024-02-06T22:24:35.890Z06/30 Read block 362 len 16384
208992024-02-06T22:24:35.891Z07/30 Read block 180 len 16384
209002024-02-06T22:24:35.891Z08/30 Write block 365 len 40960 data: 16 12 14 14 13 14 12 10 13 14
209012024-02-06T22:24:35.891Z09/30 Read block 35 len 40960
209022024-02-06T22:24:35.894Z10/30 Write block 194 len 8192 data: 15 13
209032024-02-06T22:24:35.894Z11/30 Read block 78 len 12288
209042024-02-06T22:24:35.894Z12/30 Write block 555 len 40960 data: 16 16 14 15 12 12 14 15 13 15
209052024-02-06T22:24:35.894Z13/30 Flush
209062024-02-06T22:24:35.897Z14/30 Write block 464 len 8192 data: 15 15
209072024-02-06T22:24:35.897Z15/30 Flush
209082024-02-06T22:24:35.900Z16/30 Write block 320 len 32768 data: 13 15 17 14 12 10 9 10
209092024-02-06T22:24:35.900Z17/30 Write block 293 len 12288 data: 13 12 15
209102024-02-06T22:24:35.900Z18/30 Read block 578 len 20480
209112024-02-06T22:24:35.903Z19/30 Flush
209122024-02-06T22:24:35.903Z20/30 Write block 354 len 24576 data: 19 21 19 20 19 15
209132024-02-06T22:24:35.906Z21/30 Write block 89 len 20480 data: 14 13 11 6 7
209142024-02-06T22:24:35.906Z22/30 Write block 378 len 32768 data: 15 12 15 16 13 14 14 15
209152024-02-06T22:24:35.906Z23/30 Read block 569 len 28672
209162024-02-06T22:24:35.909Z24/30 Write block 345 len 16384 data: 14 15 16 19
209172024-02-06T22:24:35.909Z25/30 Write block 467 len 36864 data: 19 18 14 13 13 13 12 12 13
209182024-02-06T22:24:35.909Z26/30 Write block 439 len 8192 data: 16 14
209192024-02-06T22:24:35.909Z27/30 Read block 306 len 12288
209202024-02-06T22:24:35.912Z28/30 Read block 467 len 24576
209212024-02-06T22:24:35.913Z29/30 Write block 33 len 4096 data: 10
209222024-02-06T22:24:35.913Z30/30 Read block 114 len 36864
209232024-02-06T22:24:35.913Z----------------------------------------------------------------
209242024-02-06T22:24:35.913Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
209252024-02-06T22:24:35.913ZStates: Active Active Active
209262024-02-06T22:24:35.913ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209272024-02-06T22:24:35.913Z 1 Acked 1000 Write 7 Done Done Done false
209282024-02-06T22:24:35.913Z 2 Acked 1001 Read 9 Done Done Done false
209292024-02-06T22:24:35.913Z 3 Acked 1002 Read 10 Done Done Done false
209302024-02-06T22:24:35.913Z 4 Acked 1003 Read 6 Sent Done Done false
209312024-02-06T22:24:35.913Z 5 Acked 1004 Write 8 Sent Done Done false
209322024-02-06T22:24:35.913Z 6 Acked 1005 Read 4 Sent Done Done false
209332024-02-06T22:24:35.913Z 7 Acked 1006 Read 4 Sent Done Done false
209342024-02-06T22:24:35.913Z 8 Acked 1007 Write 10 Sent Done Done false
209352024-02-06T22:24:35.913Z 9 Acked 1008 Read 10 Sent Done Done false
209362024-02-06T22:24:35.913Z 10 Acked 1009 Write 2 Sent Done Done false
209372024-02-06T22:24:35.913Z 11 Acked 1010 Read 3 Sent Done Done false
209382024-02-06T22:24:35.913Z 12 Acked 1011 Write 10 Sent Done Done false
209392024-02-06T22:24:35.913Z 13 Acked 1012 Flush 0 Sent Done Done false
209402024-02-06T22:24:35.913Z 14 Acked 1013 Write 2 Sent Done Done false
209412024-02-06T22:24:35.913Z 15 Acked 1014 Flush 0 Sent Done Done false
209422024-02-06T22:24:35.913Z 16 Acked 1015 Write 8 Sent Done Done false
209432024-02-06T22:24:35.913Z 17 Acked 1016 Write 3 Sent Done Done false
209442024-02-06T22:24:35.914Z 18 Acked 1017 Read 5 Sent Done Done false
209452024-02-06T22:24:35.914Z 19 Acked 1018 Flush 0 Sent Done Done false
209462024-02-06T22:24:35.914Z 20 Acked 1019 Write 6 Sent Done Done false
209472024-02-06T22:24:35.914Z 21 Acked 1020 Write 5 Sent Done Done false
209482024-02-06T22:24:35.914Z 22 Acked 1021 Write 8 Sent Done Done false
209492024-02-06T22:24:35.914Z 23 Acked 1022 Read 7 Sent Done Done false
209502024-02-06T22:24:35.914Z 24 Acked 1023 Write 4 Sent Done Done false
209512024-02-06T22:24:35.914Z 25 Acked 1024 Write 9 Sent Done Done false
209522024-02-06T22:24:35.914Z 26 Acked 1025 Write 2 Sent Done Done false
209532024-02-06T22:24:35.914Z 27 Acked 1026 Read 3 Sent Done Done false
209542024-02-06T22:24:35.914Z 28 Acked 1027 Read 6 Sent Done Done false
209552024-02-06T22:24:35.914Z 29 Acked 1028 Write 1 Sent Done Done false
209562024-02-06T22:24:35.914Z 30 Acked 1029 Read 9 Sent Done Sent false
209572024-02-06T22:24:35.914Z STATES DS:0 DS:1 DS:2 TOTAL
209582024-02-06T22:24:35.914Z New 0 0 0 0
209592024-02-06T22:24:35.914Z Sent 27 0 1 28
209602024-02-06T22:24:35.914Z Done 3 30 29 62
209612024-02-06T22:24:35.914Z Skipped 0 0 0 0
209622024-02-06T22:24:35.914Z Error 0 0 0 0
209632024-02-06T22:24:35.914ZLast Flush: 0 1018 1018
209642024-02-06T22:24:35.914ZDownstairs last five completed:
209652024-02-06T22:24:35.914ZUpstairs last five completed: 30 29 28 27 26
209662024-02-06T22:24:35.914Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:35.078238869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1909}
209672024-02-06T22:24:35.914ZWrote out file "/tmp/test_repair_verify.data"
209682024-02-06T22:24:35.918Z
209692024-02-06T22:24:35.921ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209702024-02-06T22:24:35.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209712024-02-06T22:24:35.937Z 0 000-019 185 185 185 330 330 330 F F F
209722024-02-06T22:24:35.937Z 1 020-039 190 190 190 337 337 337 F T T <---
209732024-02-06T22:24:35.937Z 2 040-059 191 191 191 340 340 340 F F F
209742024-02-06T22:24:35.937Z 3 060-079 190 190 190 339 339 339 F F F
209752024-02-06T22:24:35.937Z 4 080-099 186 186 186 331 331 331 F T T <---
209762024-02-06T22:24:35.941Z 5 100-119 190 190 190 338 338 338 F F F
209772024-02-06T22:24:35.941Z 6 120-139 186 186 186 331 331 331 F F F
209782024-02-06T22:24:35.941Z 7 140-159 190 190 190 337 337 337 F F F
209792024-02-06T22:24:35.941Z 8 160-179 184 184 184 329 329 329 F F F
209802024-02-06T22:24:35.941Z 9 180-199 190 192 192 339 341 341 F F F <---
209812024-02-06T22:24:35.941Z 10 200-219 190 190 190 339 339 339 F F F
209822024-02-06T22:24:35.941Z 11 220-239 190 190 190 337 337 337 F F F
209832024-02-06T22:24:35.941Z 12 240-259 189 189 189 335 335 335 F F F
209842024-02-06T22:24:35.941Z 13 260-279 186 186 186 331 331 331 F F F
209852024-02-06T22:24:35.941Z 14 280-299 186 192 192 331 343 343 F F F <---
209862024-02-06T22:24:35.941Z 15 300-319 189 189 189 335 335 335 F F F
209872024-02-06T22:24:35.941Z 16 320-339 186 192 192 331 343 343 F F F <---
209882024-02-06T22:24:35.941Z 17 340-359 177 177 177 322 322 322 F T T <---
209892024-02-06T22:24:35.941Z 18 360-379 190 192 192 339 341 341 F T T <---
209902024-02-06T22:24:35.941Z 19 380-399 182 182 182 327 327 327 F T T <---
209912024-02-06T22:24:35.941Z 20 400-419 190 190 190 339 339 339 F F F
209922024-02-06T22:24:35.941Z 21 420-439 185 185 185 330 330 330 F T T <---
209932024-02-06T22:24:35.941Z 22 440-459 189 192 192 335 341 341 T T T <---
209942024-02-06T22:24:35.941Z 23 460-479 190 192 192 338 342 342 F T T <---
209952024-02-06T22:24:35.942Z 24 480-499 191 191 191 340 340 340 F F F
209962024-02-06T22:24:35.942Z 25 500-519 190 190 190 339 339 339 F F F
209972024-02-06T22:24:35.942Z 26 520-539 191 191 191 340 340 340 F F F
209982024-02-06T22:24:35.942Z 27 540-559 188 192 192 333 341 341 F F F <---
209992024-02-06T22:24:35.942Z 28 560-579 186 192 192 331 341 341 F F F <---
210002024-02-06T22:24:35.942Z 29 580-599 183 183 183 328 328 328 F F F
210012024-02-06T22:24:35.942ZMax gen: 192, Max flush: 343
210022024-02-06T22:24:35.942ZError: Difference in extent metadata found!
210032024-02-06T22:24:35.942ZA Difference in extent metadata is expected here
210042024-02-06T22:24:35.942ZOn loop 96
210052024-02-06T22:24:35.942Z
210062024-02-06T22:24:35.945ZVerifying data now
210072024-02-06T22:24:40.588ZNo change after verify
210082024-02-06T22:24:40.589ZLoop: 96 Downstairs dump after verify (and repair):
210092024-02-06T22:24:40.604ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210102024-02-06T22:24:40.604Z 0 000-019 185 185 185 330 330 330 F F F
210112024-02-06T22:24:40.604Z 1 020-039 193 193 193 344 344 344 F F F
210122024-02-06T22:24:40.604Z 2 040-059 191 191 191 340 340 340 F F F
210132024-02-06T22:24:40.604Z 3 060-079 190 190 190 339 339 339 F F F
210142024-02-06T22:24:40.604Z 4 080-099 193 193 193 344 344 344 F F F
210152024-02-06T22:24:40.604Z 5 100-119 190 190 190 338 338 338 F F F
210162024-02-06T22:24:40.604Z 6 120-139 186 186 186 331 331 331 F F F
210172024-02-06T22:24:40.604Z 7 140-159 190 190 190 337 337 337 F F F
210182024-02-06T22:24:40.608Z 8 160-179 184 184 184 329 329 329 F F F
210192024-02-06T22:24:40.608Z 9 180-199 192 192 192 341 341 341 F F F
210202024-02-06T22:24:40.608Z 10 200-219 190 190 190 339 339 339 F F F
210212024-02-06T22:24:40.608Z 11 220-239 190 190 190 337 337 337 F F F
210222024-02-06T22:24:40.608Z 12 240-259 189 189 189 335 335 335 F F F
210232024-02-06T22:24:40.608Z 13 260-279 186 186 186 331 331 331 F F F
210242024-02-06T22:24:40.608Z 14 280-299 192 192 192 343 343 343 F F F
210252024-02-06T22:24:40.608Z 15 300-319 189 189 189 335 335 335 F F F
210262024-02-06T22:24:40.608Z 16 320-339 192 192 192 343 343 343 F F F
210272024-02-06T22:24:40.608Z 17 340-359 193 193 193 344 344 344 F F F
210282024-02-06T22:24:40.608Z 18 360-379 193 193 193 344 344 344 F F F
210292024-02-06T22:24:40.608Z 19 380-399 193 193 193 344 344 344 F F F
210302024-02-06T22:24:40.609Z 20 400-419 190 190 190 339 339 339 F F F
210312024-02-06T22:24:40.609Z 21 420-439 193 193 193 344 344 344 F F F
210322024-02-06T22:24:40.609Z 22 440-459 193 193 193 344 344 344 F F F
210332024-02-06T22:24:40.609Z 23 460-479 193 193 193 344 344 344 F F F
210342024-02-06T22:24:40.609Z 24 480-499 191 191 191 340 340 340 F F F
210352024-02-06T22:24:40.609Z 25 500-519 190 190 190 339 339 339 F F F
210362024-02-06T22:24:40.609Z 26 520-539 191 191 191 340 340 340 F F F
210372024-02-06T22:24:40.609Z 27 540-559 192 192 192 341 341 341 F F F
210382024-02-06T22:24:40.609Z 28 560-579 192 192 192 341 341 341 F F F
210392024-02-06T22:24:40.609Z 29 580-599 183 183 183 328 328 328 F F F
210402024-02-06T22:24:40.609ZMax gen: 193, Max flush: 344
210412024-02-06T22:24:40.609Z
210422024-02-06T22:24:40.609ZBegin loop 97
210432024-02-06T22:24:40.609ZDownstairs to restart: 1
210442024-02-06T22:24:40.625Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789167825Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917}
210452024-02-06T22:24:40.625Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
210462024-02-06T22:24:40.630Z debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789454677Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917}
210472024-02-06T22:24:40.630Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789513809Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917}
210482024-02-06T22:24:40.630Z{"msg":"Crucible c91ed082-ca0c-4cb7-a53c-a1ae84d90e2a has session id: cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789558033Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210492024-02-06T22:24:40.630Z{"msg":"Upstairs opts: Upstairs UUID: c91ed082-ca0c-4cb7-a53c-a1ae84d90e2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789617285Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210502024-02-06T22:24:40.630Z{"msg":"Crucible stats registered with UUID: c91ed082-ca0c-4cb7-a53c-a1ae84d90e2a","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789659129Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210512024-02-06T22:24:40.630Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789687885Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917}
210522024-02-06T22:24:40.630Z{"msg":"c91ed082-ca0c-4cb7-a53c-a1ae84d90e2a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789719481Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210532024-02-06T22:24:40.630Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789747987Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210542024-02-06T22:24:40.630Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789777763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210552024-02-06T22:24:40.631Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789817798Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210562024-02-06T22:24:40.631Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789853523Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210572024-02-06T22:24:40.631Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789883629Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210582024-02-06T22:24:40.631Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789940881Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210592024-02-06T22:24:40.631Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789969297Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210602024-02-06T22:24:40.631Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.789996103Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210612024-02-06T22:24:40.631Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790108998Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210622024-02-06T22:24:40.631Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:39.790174929Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210632024-02-06T22:24:40.631Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:39.790258668Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210642024-02-06T22:24:40.631Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790300882Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210652024-02-06T22:24:40.631Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790348166Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210662024-02-06T22:24:40.632Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:39.79039512Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210672024-02-06T22:24:40.632Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790420706Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210682024-02-06T22:24:40.632Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790452322Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210692024-02-06T22:24:40.632Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790486337Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210702024-02-06T22:24:40.632Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790843249Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210712024-02-06T22:24:40.632Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790887233Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210722024-02-06T22:24:40.632Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790916989Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210732024-02-06T22:24:40.632Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.790963703Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210742024-02-06T22:24:40.632Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.791423381Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210752024-02-06T22:24:40.632Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.791560152Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210762024-02-06T22:24:40.632Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.791981325Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210772024-02-06T22:24:40.632Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.79201763Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210782024-02-06T22:24:40.632Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.792038018Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210792024-02-06T22:24:40.632Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.792067994Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210802024-02-06T22:24:40.632Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.792101369Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210812024-02-06T22:24:40.632Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:39.792123806Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210822024-02-06T22:24:41.626Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.790812681Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210832024-02-06T22:24:41.626Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.790880332Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210842024-02-06T22:24:41.631Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.791695162Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210852024-02-06T22:24:41.631Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.791746465Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210862024-02-06T22:24:41.631Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792164589Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210872024-02-06T22:24:41.631Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792662551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210882024-02-06T22:24:41.631Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792698157Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210892024-02-06T22:24:41.631Z{"msg":"[0]R flush_numbers[0..12]: [330, 344, 340, 339, 344, 338, 331, 337, 329, 341, 339, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792715334Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210902024-02-06T22:24:41.631Z{"msg":"[0]R generation[0..12]: [185, 193, 191, 190, 193, 190, 186, 190, 184, 192, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792731212Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210912024-02-06T22:24:41.631Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792753869Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210922024-02-06T22:24:41.631Z{"msg":"[1]R flush_numbers[0..12]: [330, 344, 340, 339, 344, 338, 331, 337, 329, 341, 339, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792769627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210932024-02-06T22:24:41.631Z{"msg":"[1]R generation[0..12]: [185, 193, 191, 190, 193, 190, 186, 190, 184, 192, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792790864Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210942024-02-06T22:24:41.631Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792806312Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210952024-02-06T22:24:41.631Z{"msg":"[2]R flush_numbers[0..12]: [330, 344, 340, 339, 344, 338, 331, 337, 329, 341, 339, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792827519Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210962024-02-06T22:24:41.631Z{"msg":"[2]R generation[0..12]: [185, 193, 191, 190, 193, 190, 186, 190, 184, 192, 190, 190]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
210972024-02-06T22:24:41.632Z,"time":"2024-02-06T22:24:40.792842477Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210982024-02-06T22:24:41.632Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792917487Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
210992024-02-06T22:24:41.632Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792942593Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211002024-02-06T22:24:41.632Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792957871Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211012024-02-06T22:24:41.632Z{"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.7929715Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211022024-02-06T22:24:41.632Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.792992637Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211032024-02-06T22:24:41.632Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793006935Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211042024-02-06T22:24:41.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793020223Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211052024-02-06T22:24:41.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793034101Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211062024-02-06T22:24:41.632Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793055578Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211072024-02-06T22:24:41.632Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793070136Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211082024-02-06T22:24:41.632Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793082934Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211092024-02-06T22:24:41.632Z{"msg":"c91ed082-ca0c-4cb7-a53c-a1ae84d90e2a is now active with session: cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793103522Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211102024-02-06T22:24:41.632Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.7931169Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917,"session_id":"cd88e4e8-4b05-4d73-b0f6-d5faab4d7e79"}
211112024-02-06T22:24:41.632Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:40.793130678Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917}
211122024-02-06T22:24:41.632ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211132024-02-06T22:24:41.632ZLoading write count information from file "/tmp/test_repair_verify.data"
211142024-02-06T22:24:41.632ZRun Repair workload
211152024-02-06T22:24:41.632Z01/30 Write block 0 len 32768 data: 4 7 9 12 11 10 9 10
211162024-02-06T22:24:41.636Z02/30 Read block 448 len 24576
211172024-02-06T22:24:41.636Z03/30 Read block 90 len 20480
211182024-02-06T22:24:41.636Z04/30 Read block 45 len 20480
211192024-02-06T22:24:41.636Z05/30 Flush
211202024-02-06T22:24:41.636Z06/30 Write block 130 len 4096 data: 19
211212024-02-06T22:24:41.639Z07/30 Read block 92 len 32768
211222024-02-06T22:24:41.639Z08/30 Flush
211232024-02-06T22:24:41.639Z09/30 Write block 534 len 16384 data: 15 14 13 12
211242024-02-06T22:24:41.642Z10/30 Write block 129 len 40960 data: 17 20 17 15 17 15 14 11 11 13
211252024-02-06T22:24:41.642Z11/30 Flush
211262024-02-06T22:24:41.645Z12/30 Write block 158 len 40960 data: 16 19 17 18 21 18 17 13 15 18
211272024-02-06T22:24:41.645Z13/30 Read block 276 len 4096
211282024-02-06T22:24:41.649Z14/30 Read block 119 len 28672
211292024-02-06T22:24:41.649Z15/30 Read block 249 len 28672
211302024-02-06T22:24:41.649Z16/30 Read block 364 len 32768
211312024-02-06T22:24:41.649Z17/30 Flush
211322024-02-06T22:24:41.652Z18/30 Write block 393 len 20480 data: 11 10 9 8 9
211332024-02-06T22:24:41.652Z19/30 Write block 405 len 24576 data: 16 17 15 14 16 15
211342024-02-06T22:24:41.652Z20/30 Read block 238 len 12288
211352024-02-06T22:24:41.655Z21/30 Flush
211362024-02-06T22:24:41.655Z22/30 Write block 102 len 12288 data: 11 9 7
211372024-02-06T22:24:41.655Z23/30 Flush
211382024-02-06T22:24:41.658Z24/30 Write block 584 len 24576 data: 13 15 14 16 17 20
211392024-02-06T22:24:41.658Z25/30 Flush
211402024-02-06T22:24:41.661Z26/30 Write block 564 len 8192 data: 16 15
211412024-02-06T22:24:41.661Z27/30 Write block 0 len 20480 data: 5 8 10 13 12
211422024-02-06T22:24:41.661Z28/30 Write block 235 len 24576 data: 20 18 16 11 11 9
211432024-02-06T22:24:41.665Z29/30 Write block 277 len 12288 data: 16 21 22
211442024-02-06T22:24:41.665Z30/30 Write block 272 len 20480 data: 16 12 10 14 14
211452024-02-06T22:24:41.665Z----------------------------------------------------------------
211462024-02-06T22:24:41.665Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
211472024-02-06T22:24:41.665ZStates: Active Active Active
211482024-02-06T22:24:41.665ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211492024-02-06T22:24:41.665Z 1 Acked 1000 Write 8 Done Done Done false
211502024-02-06T22:24:41.665Z 2 Acked 1001 Read 6 Done Done Done false
211512024-02-06T22:24:41.665Z 3 Acked 1002 Read 5 Done Sent Done false
211522024-02-06T22:24:41.665Z 4 Acked 1003 Read 5 Done Sent Done false
211532024-02-06T22:24:41.665Z 5 Acked 1004 Flush 0 Done Sent Done false
211542024-02-06T22:24:41.665Z 6 Acked 1005 Write 1 Done Sent Done false
211552024-02-06T22:24:41.665Z 7 Acked 1006 Read 8 Done Sent Done false
211562024-02-06T22:24:41.666Z 8 Acked 1007 Flush 0 Done Sent Done false
211572024-02-06T22:24:41.666Z 9 Acked 1008 Write 4 Done Sent Done false
211582024-02-06T22:24:41.666Z 10 Acked 1009 Write 10 Done Sent Done false
211592024-02-06T22:24:41.666Z 11 Acked 1010 Flush 0 Done Sent Done false
211602024-02-06T22:24:41.666Z 12 Acked 1011 Write 10 Done Sent Done false
211612024-02-06T22:24:41.666Z 13 Acked 1012 Read 1 Done Sent Done false
211622024-02-06T22:24:41.666Z 14 Acked 1013 Read 7 Done Sent Done false
211632024-02-06T22:24:41.666Z 15 Acked 1014 Read 7 Done Sent Done false
211642024-02-06T22:24:41.666Z 16 Acked 1015 Read 8 Done Sent Done false
211652024-02-06T22:24:41.666Z 17 Acked 1016 Flush 0 Done Sent Done false
211662024-02-06T22:24:41.666Z 18 Acked 1017 Write 5 Done Sent Done false
211672024-02-06T22:24:41.666Z 19 Acked 1018 Write 6 Done Sent Done false
211682024-02-06T22:24:41.666Z 20 Acked 1019 Read 3 Done Sent Done false
211692024-02-06T22:24:41.666Z 21 Acked 1020 Flush 0 Done Sent Done false
211702024-02-06T22:24:41.666Z 22 Acked 1021 Write 3 Done Sent Done false
211712024-02-06T22:24:41.666Z 23 Acked 1022 Flush 0 Done Sent Done false
211722024-02-06T22:24:41.666Z 24 Acked 1023 Write 6 Done Sent Done false
211732024-02-06T22:24:41.666Z 25 Acked 1024 Flush 0 Done Sent Done false
211742024-02-06T22:24:41.666Z 26 Acked 1025 Write 2 Done Sent Done false
211752024-02-06T22:24:41.666Z 27 Acked 1026 Write 5 Done Sent Done false
211762024-02-06T22:24:41.666Z 28 Acked 1027 Write 6 Done Sent Done false
211772024-02-06T22:24:41.666Z 29 Acked 1028 Write 3 Done Sent Done false
211782024-02-06T22:24:41.666Z 30 Acked 1029 Write 5 Sent Sent Sent false
211792024-02-06T22:24:41.666Z STATES DS:0 DS:1 DS:2 TOTAL
211802024-02-06T22:24:41.666Z New 0 0 0 0
211812024-02-06T22:24:41.667Z Sent 1 28 1 30
211822024-02-06T22:24:41.667Z Done 29 2 29 60
211832024-02-06T22:24:41.667Z Skipped 0 0 0 0
211842024-02-06T22:24:41.667Z Error 0 0 0 0
211852024-02-06T22:24:41.667ZLast Flush: 1024 0 1024
211862024-02-06T22:24:41.667ZDownstairs last five completed:
211872024-02-06T22:24:41.667ZUpstairs last five completed: 30 29 28 27 26
211882024-02-06T22:24:41.667Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:40.828278609Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1917}
211892024-02-06T22:24:41.667ZWrote out file "/tmp/test_repair_verify.data"
211902024-02-06T22:24:41.691Z
211912024-02-06T22:24:41.691ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211922024-02-06T22:24:41.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211932024-02-06T22:24:41.691Z 0 000-019 194 185 194 345 330 345 T T T <---
211942024-02-06T22:24:41.691Z 1 020-039 193 193 193 344 344 344 F F F
211952024-02-06T22:24:41.691Z 2 040-059 191 191 191 340 340 340 F F F
211962024-02-06T22:24:41.691Z 3 060-079 190 190 190 339 339 339 F F F
211972024-02-06T22:24:41.691Z 4 080-099 193 193 193 344 344 344 F F F
211982024-02-06T22:24:41.691Z 5 100-119 194 190 194 350 338 350 F F F <---
211992024-02-06T22:24:41.691Z 6 120-139 194 186 194 347 331 347 F F F <---
212002024-02-06T22:24:41.691Z 7 140-159 194 190 194 348 337 348 F F F <---
212012024-02-06T22:24:41.691Z 8 160-179 194 184 194 348 329 348 F F F <---
212022024-02-06T22:24:41.691Z 9 180-199 192 192 192 341 341 341 F F F
212032024-02-06T22:24:41.691Z 10 200-219 190 190 190 339 339 339 F F F
212042024-02-06T22:24:41.691Z 11 220-239 190 190 190 337 337 337 T F T <---
212052024-02-06T22:24:41.691Z 12 240-259 189 189 189 335 335 335 T F T <---
212062024-02-06T22:24:41.691Z 13 260-279 186 186 186 331 331 331 T F T <---
212072024-02-06T22:24:41.692Z 14 280-299 192 192 192 343 343 343 F F F
212082024-02-06T22:24:41.692Z 15 300-319 189 189 189 335 335 335 F F F
212092024-02-06T22:24:41.692Z 16 320-339 192 192 192 343 343 343 F F F
212102024-02-06T22:24:41.692Z 17 340-359 193 193 193 344 344 344 F F F
212112024-02-06T22:24:41.692Z 18 360-379 193 193 193 344 344 344 F F F
212122024-02-06T22:24:41.692Z 19 380-399 194 193 194 349 344 349 F F F <---
212132024-02-06T22:24:41.692Z 20 400-419 194 190 194 349 339 349 F F F <---
212142024-02-06T22:24:41.692Z 21 420-439 193 193 193 344 344 344 F F F
212152024-02-06T22:24:41.692Z 22 440-459 193 193 193 344 344 344 F F F
212162024-02-06T22:24:41.692Z 23 460-479 193 193 193 344 344 344 F F F
212172024-02-06T22:24:41.692Z 24 480-499 191 191 191 340 340 340 F F F
212182024-02-06T22:24:41.692Z 25 500-519 190 190 190 339 339 339 F F F
212192024-02-06T22:24:41.692Z 26 520-539 194 191 194 347 340 347 F F F <---
212202024-02-06T22:24:41.692Z 27 540-559 192 192 192 341 341 341 F F F
212212024-02-06T22:24:41.692Z 28 560-579 192 192 192 341 341 341 T F T <---
212222024-02-06T22:24:41.692Z 29 580-599 194 183 194 351 328 351 F F F <---
212232024-02-06T22:24:41.692ZMax gen: 194, Max flush: 351
212242024-02-06T22:24:41.692ZError: Difference in extent metadata found!
212252024-02-06T22:24:41.692ZA Difference in extent metadata is expected here
212262024-02-06T22:24:41.692ZOn loop 97
212272024-02-06T22:24:41.692Z
212282024-02-06T22:24:41.697ZVerifying data now
212292024-02-06T22:24:46.332ZNo change after verify
212302024-02-06T22:24:46.333ZLoop: 97 Downstairs dump after verify (and repair):
212312024-02-06T22:24:46.348ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212322024-02-06T22:24:46.348Z 0 000-019 195 195 195 352 352 352 F F F
212332024-02-06T22:24:46.348Z 1 020-039 193 193 193 344 344 344 F F F
212342024-02-06T22:24:46.348Z 2 040-059 191 191 191 340 340 340 F F F
212352024-02-06T22:24:46.348Z 3 060-079 190 190 190 339 339 339 F F F
212362024-02-06T22:24:46.348Z 4 080-099 193 193 193 344 344 344 F F F
212372024-02-06T22:24:46.348Z 5 100-119 194 194 194 350 350 350 F F F
212382024-02-06T22:24:46.348Z 6 120-139 194 194 194 347 347 347 F F F
212392024-02-06T22:24:46.348Z 7 140-159 194 194 194 348 348 348 F F F
212402024-02-06T22:24:46.348Z 8 160-179 194 194 194 348 348 348 F F F
212412024-02-06T22:24:46.348Z 9 180-199 192 192 192 341 341 341 F F F
212422024-02-06T22:24:46.352Z 10 200-219 190 190 190 339 339 339 F F F
212432024-02-06T22:24:46.352Z 11 220-239 195 195 195 352 352 352 F F F
212442024-02-06T22:24:46.353Z 12 240-259 195 195 195 352 352 352 F F F
212452024-02-06T22:24:46.353Z 13 260-279 195 195 195 352 352 352 F F F
212462024-02-06T22:24:46.353Z 14 280-299 192 192 192 343 343 343 F F F
212472024-02-06T22:24:46.353Z 15 300-319 189 189 189 335 335 335 F F F
212482024-02-06T22:24:46.353Z 16 320-339 192 192 192 343 343 343 F F F
212492024-02-06T22:24:46.353Z 17 340-359 193 193 193 344 344 344 F F F
212502024-02-06T22:24:46.353Z 18 360-379 193 193 193 344 344 344 F F F
212512024-02-06T22:24:46.353Z 19 380-399 194 194 194 349 349 349 F F F
212522024-02-06T22:24:46.353Z 20 400-419 194 194 194 349 349 349 F F F
212532024-02-06T22:24:46.353Z 21 420-439 193 193 193 344 344 344 F F F
212542024-02-06T22:24:46.353Z 22 440-459 193 193 193 344 344 344 F F F
212552024-02-06T22:24:46.353Z 23 460-479 193 193 193 344 344 344 F F F
212562024-02-06T22:24:46.353Z 24 480-499 191 191 191 340 340 340 F F F
212572024-02-06T22:24:46.353Z 25 500-519 190 190 190 339 339 339 F F F
212582024-02-06T22:24:46.353Z 26 520-539 194 194 194 347 347 347 F F F
212592024-02-06T22:24:46.353Z 27 540-559 192 192 192 341 341 341 F F F
212602024-02-06T22:24:46.353Z 28 560-579 195 195 195 352 352 352 F F F
212612024-02-06T22:24:46.353Z 29 580-599 194 194 194 351 351 351 F F F
212622024-02-06T22:24:46.353ZMax gen: 195, Max flush: 352
212632024-02-06T22:24:46.353Z
212642024-02-06T22:24:46.353ZBegin loop 98
212652024-02-06T22:24:46.353ZDownstairs to restart: 2
212662024-02-06T22:24:46.369Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533178011Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925}
212672024-02-06T22:24:46.369Z{"msg":"Crucible Version: BuildInfo {\n version: \"Crucible runtime is spawned
212682024-02-06T22:24:46.379Z0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533500307Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925}
212692024-02-06T22:24:46.379Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533549041Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925}
212702024-02-06T22:24:46.379Z{"msg":"Crucible 53628288-4aa6-4ab7-93f7-fe5d2581c22e has session id: aa1995ce-d9ad-4edd-881b-1ee1e97e0f11","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533594854Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212712024-02-06T22:24:46.380Z{"msg":"Upstairs opts: Upstairs UUID: 53628288-4aa6-4ab7-93f7-fe5d2581c22e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533652567Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212722024-02-06T22:24:46.380Z{"msg":"Crucible stats registered with UUID: 53628288-4aa6-4ab7-93f7-fe5d2581c22e","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.53369788Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212732024-02-06T22:24:46.380Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533744324Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925}
212742024-02-06T22:24:46.380Z{"msg":"53628288-4aa6-4ab7-93f7-fe5d2581c22e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533828923Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212752024-02-06T22:24:46.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533877256Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212762024-02-06T22:24:46.380Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533915221Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212772024-02-06T22:24:46.381Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.533960975Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212782024-02-06T22:24:46.381Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534002149Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212792024-02-06T22:24:46.381Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534037245Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212802024-02-06T22:24:46.381Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534077049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212812024-02-06T22:24:46.381Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534115234Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212822024-02-06T22:24:46.381Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534160058Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212832024-02-06T22:24:46.381Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534187304Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212842024-02-06T22:24:46.381Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534226789Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212852024-02-06T22:24:46.382Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534268953Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212862024-02-06T22:24:46.382Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:45.534302049Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212872024-02-06T22:24:46.382Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:45.534341784Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212882024-02-06T22:24:46.382Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:45.534384598Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212892024-02-06T22:24:46.382Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534411214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212902024-02-06T22:24:46.382Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534454148Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212912024-02-06T22:24:46.382Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534481105Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212922024-02-06T22:24:46.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534798302Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212932024-02-06T22:24:46.382Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534849665Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212942024-02-06T22:24:46.382Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534928324Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212952024-02-06T22:24:46.383Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.534954561Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212962024-02-06T22:24:46.383Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.535317232Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212972024-02-06T22:24:46.383Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.535488179Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212982024-02-06T22:24:46.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.535890274Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
212992024-02-06T22:24:46.383Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.535932129Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213002024-02-06T22:24:46.383Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.535968214Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213012024-02-06T22:24:46.383Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.536017837Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213022024-02-06T22:24:46.383Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.536051442Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213032024-02-06T22:24:46.383Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:45.5360723Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213042024-02-06T22:24:47.372Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.535624848Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213052024-02-06T22:24:47.372Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.535738193Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213062024-02-06T22:24:47.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.53650452Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213072024-02-06T22:24:47.377Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.536543034Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213082024-02-06T22:24:47.377Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537003432Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213092024-02-06T22:24:47.377Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.53753714Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213102024-02-06T22:24:47.377Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537572195Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213112024-02-06T22:24:47.377Z{"msg":"[0]R flush_numbers[0..12]: [352, 344, 340, 339, 344, 350, 347, 348, 348, 341, 339, 352]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537589673Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213122024-02-06T22:24:47.377Z{"msg":"[0]R generation[0..12]: [195, 193, 191, 190, 193, 194, 194, 194, 194, 192, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537633057Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213132024-02-06T22:24:47.377Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537660623Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213142024-02-06T22:24:47.377Z{"msg":"[1]R flush_numbers[0..12]: [352, 344, 340, 339, 344, 350, 347, 348, 348, 341, 339, 352]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537676461Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213152024-02-06T22:24:47.377Z{"msg":"[1]R generation[0..12]: [195, 193, 191, 190, 193, 194, 194, 194, 194, 192, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537698178Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213162024-02-06T22:24:47.377Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537713176Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213172024-02-06T22:24:47.377Z{"msg":"[2]R flush_numbers[0..12]: [352, 344, 340, 339, 344, 350, 347, 348, 348, 341, 339, 352]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
213182024-02-06T22:24:47.377Z,"time":"2024-02-06T22:24:46.537735513Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213192024-02-06T22:24:47.377Z{"msg":"[2]R generation[0..12]: [195, 193, 191, 190, 193, 194, 194, 194, 194, 192, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537752891Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213202024-02-06T22:24:47.378Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537774368Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213212024-02-06T22:24:47.378Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537788636Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213222024-02-06T22:24:47.378Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537810103Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213232024-02-06T22:24:47.378Z{"msg":"Next flush: 353","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537823701Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213242024-02-06T22:24:47.378Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537843888Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213252024-02-06T22:24:47.378Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537857646Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213262024-02-06T22:24:47.378Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537877414Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213272024-02-06T22:24:47.378Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537891562Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213282024-02-06T22:24:47.378Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537911719Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213292024-02-06T22:24:47.378Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537925408Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213302024-02-06T22:24:47.378Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537957753Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213312024-02-06T22:24:47.378Z{"msg":"53628288-4aa6-4ab7-93f7-fe5d2581c22e is now active with session: aa1995ce-d9ad-4edd-881b-1ee1e97e0f11","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.537990489Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213322024-02-06T22:24:47.378Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.538005777Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925,"session_id":"aa1995ce-d9ad-4edd-881b-1ee1e97e0f11"}
213332024-02-06T22:24:47.378Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:46.538026154Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925}
213342024-02-06T22:24:47.378ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213352024-02-06T22:24:47.378ZLoading write count information from file "/tmp/test_repair_verify.data"
213362024-02-06T22:24:47.378ZRun Repair workload
213372024-02-06T22:24:47.378Z01/30 Write block 337 len 12288 data: 18 18 17
213382024-02-06T22:24:47.382Z02/30 Read block 201 len 12288
213392024-02-06T22:24:47.382Z03/30 Read block 330 len 24576
213402024-02-06T22:24:47.382Z04/30 Read block 181 len 40960
213412024-02-06T22:24:47.382Z05/30 Read block 374 len 28672
213422024-02-06T22:24:47.382Z06/30 Read block 561 len 24576
213432024-02-06T22:24:47.382Z07/30 Flush
213442024-02-06T22:24:47.385Z08/30 Write block 409 len 12288 data: 17 16 16
213452024-02-06T22:24:47.385Z09/30 Read block 176 len 4096
213462024-02-06T22:24:47.385Z10/30 Read block 338 len 8192
213472024-02-06T22:24:47.386Z11/30 Read block 20 len 4096
213482024-02-06T22:24:47.386Z12/30 Flush
213492024-02-06T22:24:47.389Z13/30 Write block 76 len 8192 data: 12 13
213502024-02-06T22:24:47.389Z14/30 Write block 43 len 8192 data: 13 13
213512024-02-06T22:24:47.389Z15/30 Read block 438 len 16384
213522024-02-06T22:24:47.392Z16/30 Write block 463 len 12288 data: 14 16 16
213532024-02-06T22:24:47.392Z17/30 Read block 340 len 8192
213542024-02-06T22:24:47.392Z18/30 Read block 381 len 24576
213552024-02-06T22:24:47.392Z19/30 Read block 32 len 36864
213562024-02-06T22:24:47.395Z20/30 Write block 37 len 28672 data: 11 13 12 16 15 13 14
213572024-02-06T22:24:47.395Z21/30 Write block 340 len 32768 data: 17 17 21 16 16 15 16 17
213582024-02-06T22:24:47.395Z22/30 Read block 122 len 36864
213592024-02-06T22:24:47.398Z23/30 Read block 223 len 40960
213602024-02-06T22:24:47.398Z24/30 Read block 403 len 20480
213612024-02-06T22:24:47.399Z25/30 Flush
213622024-02-06T22:24:47.401Z26/30 Write block 348 len 4096 data: 20
213632024-02-06T22:24:47.402Z27/30 Write block 177 len 36864 data: 16 16 20 20 22 21 24 25 23
213642024-02-06T22:24:47.402Z28/30 Read block 276 len 28672
213652024-02-06T22:24:47.405Z29/30 Read block 394 len 4096
213662024-02-06T22:24:47.405Z30/30 Read block 402 len 24576
213672024-02-06T22:24:47.405Z----------------------------------------------------------------
213682024-02-06T22:24:47.405Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
213692024-02-06T22:24:47.405ZStates: Active Active Active
213702024-02-06T22:24:47.406ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213712024-02-06T22:24:47.406Z 1 Acked 1000 Write 3 Done Done Done false
213722024-02-06T22:24:47.406Z 2 Acked 1001 Read 3 Done Done Done false
213732024-02-06T22:24:47.406Z 3 Acked 1002 Read 6 Done Done Done false
213742024-02-06T22:24:47.406Z 4 Acked 1003 Read 10 Done Done Sent false
213752024-02-06T22:24:47.406Z 5 Acked 1004 Read 7 Done Done Sent false
213762024-02-06T22:24:47.406Z 6 Acked 1005 Read 6 Done Done Sent false
213772024-02-06T22:24:47.406Z 7 Acked 1006 Flush 0 Done Done Sent false
213782024-02-06T22:24:47.406Z 8 Acked 1007 Write 3 Done Done Sent false
213792024-02-06T22:24:47.406Z 9 Acked 1008 Read 1 Done Done Sent false
213802024-02-06T22:24:47.406Z 10 Acked 1009 Read 2 Done Done Sent false
213812024-02-06T22:24:47.406Z 11 Acked 1010 Read 1 Done Done Sent false
213822024-02-06T22:24:47.406Z 12 Acked 1011 Flush 0 Done Done Sent false
213832024-02-06T22:24:47.406Z 13 Acked 1012 Write 2 Done Done Sent false
213842024-02-06T22:24:47.406Z 14 Acked 1013 Write 2 Done Done Sent false
213852024-02-06T22:24:47.406Z 15 Acked 1014 Read 4 Done Done Sent false
213862024-02-06T22:24:47.406Z 16 Acked 1015 Write 3 Done Done Sent false
213872024-02-06T22:24:47.406Z 17 Acked 1016 Read 2 Done Done Sent false
213882024-02-06T22:24:47.406Z 18 Acked 1017 Read 6 Done Done Sent false
213892024-02-06T22:24:47.406Z 19 Acked 1018 Read 9 Done Done Sent false
213902024-02-06T22:24:47.406Z 20 Acked 1019 Write 7 Done Done Sent false
213912024-02-06T22:24:47.406Z 21 Acked 1020 Write 8 Done Done Sent false
213922024-02-06T22:24:47.406Z 22 Acked 1021 Read 9 Done Done Sent false
213932024-02-06T22:24:47.406Z 23 Acked 1022 Read 10 Done Done Sent false
213942024-02-06T22:24:47.406Z 24 Acked 1023 Read 5 Done Done Sent false
213952024-02-06T22:24:47.407Z 25 Acked 1024 Flush 0 Done Done Sent false
213962024-02-06T22:24:47.407Z 26 Acked 1025 Write 1 Done Done Sent false
213972024-02-06T22:24:47.407Z 27 Acked 1026 Write 9 Done Done Sent false
213982024-02-06T22:24:47.407Z 28 Acked 1027 Read 7 Done Done Sent false
213992024-02-06T22:24:47.407Z 29 Acked 1028 Read 1 Done Done Sent false
214002024-02-06T22:24:47.407Z 30 Acked 1029 Read 6 Done Sent Sent false
214012024-02-06T22:24:47.407Z STATES DS:0 DS:1 DS:2 TOTAL
214022024-02-06T22:24:47.407Z New 0 0 0 0
214032024-02-06T22:24:47.407Z Sent 0 1 27 28
214042024-02-06T22:24:47.407Z Done 30 29 3 62
214052024-02-06T22:24:47.407Z Skipped 0 0 0 0
214062024-02-06T22:24:47.407Z Error 0 0 0 0
214072024-02-06T22:24:47.407ZLast Flush: 1024 1024 0
214082024-02-06T22:24:47.407ZDownstairs last five completed:
214092024-02-06T22:24:47.407ZUpstairs last five completed: 30 29 28 27 26
214102024-02-06T22:24:47.407Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:46.569060181Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1925}
214112024-02-06T22:24:47.407ZWrote out file "/tmp/test_repair_verify.data"
214122024-02-06T22:24:47.411Z
214132024-02-06T22:24:47.414ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214142024-02-06T22:24:47.429ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214152024-02-06T22:24:47.429Z 0 000-019 195 195 195 352 352 352 F F F
214162024-02-06T22:24:47.429Z 1 020-039 196 196 193 355 355 344 F F F <---
214172024-02-06T22:24:47.429Z 2 040-059 196 196 191 355 355 340 F F F <---
214182024-02-06T22:24:47.429Z 3 060-079 196 196 190 355 355 339 F F F <---
214192024-02-06T22:24:47.429Z 4 080-099 193 193 193 344 344 344 F F F
214202024-02-06T22:24:47.429Z 5 100-119 194 194 194 350 350 350 F F F
214212024-02-06T22:24:47.429Z 6 120-139 194 194 194 347 347 347 F F F
214222024-02-06T22:24:47.429Z 7 140-159 194 194 194 348 348 348 F F F
214232024-02-06T22:24:47.429Z 8 160-179 194 194 194 348 348 348 T T F <---
214242024-02-06T22:24:47.433Z 9 180-199 192 192 192 341 341 341 T T F <---
214252024-02-06T22:24:47.433Z 10 200-219 190 190 190 339 339 339 F F F
214262024-02-06T22:24:47.433Z 11 220-239 195 195 195 352 352 352 F F F
214272024-02-06T22:24:47.433Z 12 240-259 195 195 195 352 352 352 F F F
214282024-02-06T22:24:47.433Z 13 260-279 195 195 195 352 352 352 F F F
214292024-02-06T22:24:47.433Z 14 280-299 192 192 192 343 343 343 F F F
214302024-02-06T22:24:47.433Z 15 300-319 189 189 189 335 335 335 F F F
214312024-02-06T22:24:47.433Z 16 320-339 196 196 192 353 353 343 F F T <---
214322024-02-06T22:24:47.433Z 17 340-359 196 196 193 355 355 344 T T F <---
214332024-02-06T22:24:47.433Z 18 360-379 193 193 193 344 344 344 F F F
214342024-02-06T22:24:47.433Z 19 380-399 194 194 194 349 349 349 F F F
214352024-02-06T22:24:47.433Z 20 400-419 196 196 194 354 354 349 F F F <---
214362024-02-06T22:24:47.433Z 21 420-439 193 193 193 344 344 344 F F F
214372024-02-06T22:24:47.433Z 22 440-459 193 193 193 344 344 344 F F F
214382024-02-06T22:24:47.433Z 23 460-479 196 196 193 355 355 344 F F F <---
214392024-02-06T22:24:47.433Z 24 480-499 191 191 191 340 340 340 F F F
214402024-02-06T22:24:47.434Z 25 500-519 190 190 190 339 339 339 F F F
214412024-02-06T22:24:47.434Z 26 520-539 194 194 194 347 347 347 F F F
214422024-02-06T22:24:47.434Z 27 540-559 192 192 192 341 341 341 F F F
214432024-02-06T22:24:47.434Z 28 560-579 195 195 195 352 352 352 F F F
214442024-02-06T22:24:47.434Z 29 580-599 194 194 194 351 351 351 F F F
214452024-02-06T22:24:47.434ZMax gen: 196, Max flush: 355
214462024-02-06T22:24:47.434ZError: Difference in extent metadata found!
214472024-02-06T22:24:47.434ZA Difference in extent metadata is expected here
214482024-02-06T22:24:47.434ZOn loop 98
214492024-02-06T22:24:47.434Z
214502024-02-06T22:24:47.437ZVerifying data now
214512024-02-06T22:24:52.920ZNo change after verify
214522024-02-06T22:24:52.921ZLoop: 98 Downstairs dump after verify (and repair):
214532024-02-06T22:24:52.933ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214542024-02-06T22:24:52.933Z 0 000-019 195 195 195 352 352 352 F F F
214552024-02-06T22:24:52.933Z 1 020-039 196 196 196 355 355 355 F F F
214562024-02-06T22:24:52.933Z 2 040-059 196 196 196 355 355 355 F F F
214572024-02-06T22:24:52.933Z 3 060-079 196 196 196 355 355 355 F F F
214582024-02-06T22:24:52.933Z 4 080-099 193 193 193 344 344 344 F F F
214592024-02-06T22:24:52.933Z 5 100-119 194 194 194 350 350 350 F F F
214602024-02-06T22:24:52.933Z 6 120-139 194 194 194 347 347 347 F F F
214612024-02-06T22:24:52.933Z 7 140-159 194 194 194 348 348 348 F F F
214622024-02-06T22:24:52.934Z 8 160-179 197 197 197 356 356 356 F F F
214632024-02-06T22:24:52.934Z 9 180-199 197 197 197 356 356 356 F F F
214642024-02-06T22:24:52.934Z 10 200-219 190 190 190 339 339 339 F F F
214652024-02-06T22:24:52.934Z 11 220-239 195 195 195 352 352 352 F F F
214662024-02-06T22:24:52.934Z 12 240-259 195 195 195 352 352 352 F F F
214672024-02-06T22:24:52.934Z 13 260-279 195 195 195 352 352 352 F F F
214682024-02-06T22:24:52.934Z 14 280-299 192 192 192 343 343 343 F F F
214692024-02-06T22:24:52.934Z 15 300-319 189 189 189 335 335 335 F F F
214702024-02-06T22:24:52.937Z 16 320-339 196 196 196 353 353 353 F F F
214712024-02-06T22:24:52.937Z 17 340-359 197 197 197 356 356 356 F F F
214722024-02-06T22:24:52.937Z 18 360-379 193 193 193 344 344 344 F F F
214732024-02-06T22:24:52.937Z 19 380-399 194 194 194 349 349 349 F F F
214742024-02-06T22:24:52.937Z 20 400-419 196 196 196 354 354 354 F F F
214752024-02-06T22:24:52.937Z 21 420-439 193 193 193 344 344 344 F F F
214762024-02-06T22:24:52.937Z 22 440-459 193 193 193 344 344 344 F F F
214772024-02-06T22:24:52.938Z 23 460-479 196 196 196 355 355 355 F F F
214782024-02-06T22:24:52.938Z 24 480-499 191 191 191 340 340 340 F F F
214792024-02-06T22:24:52.938Z 25 500-519 190 190 190 339 339 339 F F F
214802024-02-06T22:24:52.938Z 26 520-539 194 194 194 347 347 347 F F F
214812024-02-06T22:24:52.938Z 27 540-559 192 192 192 341 341 341 F F F
214822024-02-06T22:24:52.938Z 28 560-579 195 195 195 352 352 352 F F F
214832024-02-06T22:24:52.938Z 29 580-599 194 194 194 351 351 351 F F F
214842024-02-06T22:24:52.938ZMax gen: 197, Max flush: 356
214852024-02-06T22:24:52.938Z
214862024-02-06T22:24:52.938ZBegin loop 99
214872024-02-06T22:24:52.938ZDownstairs to restart: 0
214882024-02-06T22:24:52.955Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118186626Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933}
214892024-02-06T22:24:52.955Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"7fe08e7f028451780bfe14aca37215026453fbc6\",\n git_commit_timestamp: \"2024-02-06T21:52:40.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118474947Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933}
214902024-02-06T22:24:52.959Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118496474Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933}
214912024-02-06T22:24:52.959Z{"msg":"Crucible 4bc60691-ba6a-4d18-b73b-18be2fa51f16 has session id: 14c7a522-acdd-484d-9ad7-b2ed5b9801c6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118556376Z","hostname":"Crucible runtime is spawned
214922024-02-06T22:24:52.960Zip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
214932024-02-06T22:24:52.960Z{"msg":"Upstairs opts: Upstairs UUID: 4bc60691-ba6a-4d18-b73b-18be2fa51f16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118615228Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
214942024-02-06T22:24:52.960Z{"msg":"Crucible stats registered with UUID: 4bc60691-ba6a-4d18-b73b-18be2fa51f16","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118676Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
214952024-02-06T22:24:52.960Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118716384Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933}
214962024-02-06T22:24:52.960Z{"msg":"4bc60691-ba6a-4d18-b73b-18be2fa51f16 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.11882199Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
214972024-02-06T22:24:52.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118847817Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
214982024-02-06T22:24:52.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118873423Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
214992024-02-06T22:24:52.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118914708Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215002024-02-06T22:24:52.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118940614Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215012024-02-06T22:24:52.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.118981179Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215022024-02-06T22:24:52.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119007385Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215032024-02-06T22:24:52.960Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119040731Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"io task","client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215042024-02-06T22:24:52.960Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119073056Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"io task","client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215052024-02-06T22:24:52.960Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119207408Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"io task","client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215062024-02-06T22:24:52.960Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119258821Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"io task","client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215072024-02-06T22:24:52.960Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119286238Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"io task","client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215082024-02-06T22:24:52.960Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119325342Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"io task","client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215092024-02-06T22:24:52.960Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119943439Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215102024-02-06T22:24:52.960Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119979774Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215112024-02-06T22:24:52.960Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.119999711Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215122024-02-06T22:24:52.960Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.120024268Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215132024-02-06T22:24:52.960Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.120581823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215142024-02-06T22:24:52.960Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.120737102Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215152024-02-06T22:24:52.961Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121059688Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215162024-02-06T22:24:52.961Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121112551Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215172024-02-06T22:24:52.961Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121168813Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215182024-02-06T22:24:52.961Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121200739Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215192024-02-06T22:24:52.961Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121228915Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215202024-02-06T22:24:52.961Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121259581Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215212024-02-06T22:24:52.961Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121275899Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215222024-02-06T22:24:52.961Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121297716Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215232024-02-06T22:24:52.961Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.121696032Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215242024-02-06T22:24:52.965Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122133533Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215252024-02-06T22:24:52.965Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122179887Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215262024-02-06T22:24:52.965Z{"msg":"[0]R flush_numbers[0..12]: [352, 355, 355, 355, 344, 350, 347, 348, 356, 356, 339, 352]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122206673Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215272024-02-06T22:24:52.965Z{"msg":"[0]R generation[0..12]: [195, 196, 196, 196, 193, 194, 194, 194, 197, 197, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122243448Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215282024-02-06T22:24:52.965Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122279823Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215292024-02-06T22:24:52.965Z{"msg":"[1]R flush_numbers[0..12]: [352, 355, 355, 355, 344, 350, 347, 348, 356, 356, 339, 352]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122317148Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215302024-02-06T22:24:52.965Z{"msg":"[1]R generation[0..12]: [195, 196, 196, 196, 193, 194, 194, 194, 197, 197, 190, 195]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
215312024-02-06T22:24:52.965Z,"time":"2024-02-06T22:24:52.122356073Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215322024-02-06T22:24:52.965Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122395647Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215332024-02-06T22:24:52.965Z{"msg":"[2]R flush_numbers[0..12]: [352, 355, 355, 355, 344, 350, 347, 348, 356, 356, 339, 352]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122434582Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215342024-02-06T22:24:52.965Z{"msg":"[2]R generation[0..12]: [195, 196, 196, 196, 193, 194, 194, 194, 197, 197, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122473007Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215352024-02-06T22:24:52.965Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122511202Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215362024-02-06T22:24:52.966Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122546447Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215372024-02-06T22:24:52.966Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122582942Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215382024-02-06T22:24:52.966Z{"msg":"Next flush: 357","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122618627Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215392024-02-06T22:24:52.966Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122645494Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
215402024-02-06T22:24:52.966Zdownstairs","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215412024-02-06T22:24:52.966Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122687348Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215422024-02-06T22:24:52.966Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122722523Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215432024-02-06T22:24:52.966Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122755519Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215442024-02-06T22:24:52.966Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122780695Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215452024-02-06T22:24:52.966Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122802183Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215462024-02-06T22:24:52.966Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.12282466Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215472024-02-06T22:24:52.966Z{"msg":"4bc60691-ba6a-4d18-b73b-18be2fa51f16 is now active with session: 14c7a522-acdd-484d-9ad7-b2ed5b9801c6","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122839038Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215482024-02-06T22:24:52.966Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122858595Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933,"session_id":"14c7a522-acdd-484d-9ad7-b2ed5b9801c6"}
215492024-02-06T22:24:52.966Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-06T22:24:52.122872183Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933}
215502024-02-06T22:24:52.966ZLoading write count information from file "/tmp/test_repair_verify.data"
215512024-02-06T22:24:52.966ZRun Repair workload
215522024-02-06T22:24:52.966Z01/30 Write block 567 len 40960 data: 17 18 15 15 20 20 19 21 17 16
215532024-02-06T22:24:52.966Z02/30 Write block 102 len 12288 data: 12 10 8
215542024-02-06T22:24:52.966Z03/30 Read block 476 len 8192
215552024-02-06T22:24:52.966Z04/30 Read block 477 len 24576
215562024-02-06T22:24:52.966Z05/30 Write block 508 len 20480 data: 18 18 20 17 15
215572024-02-06T22:24:52.970Z06/30 Write block 559 len 12288 data: 13 13 15
215582024-02-06T22:24:52.970Z07/30 Read block 100 len 8192
215592024-02-06T22:24:52.970Z08/30 Read block 372 len 4096
215602024-02-06T22:24:52.970Z09/30 Write block 168 len 16384 data: 21 20 20 15
215612024-02-06T22:24:52.970Z10/30 Read block 505 len 36864
215622024-02-06T22:24:52.970Z11/30 Read block 75 len 32768
215632024-02-06T22:24:52.970Z12/30 Write block 193 len 32768 data: 12 16 14 16 14 12 12 12
215642024-02-06T22:24:52.973Z13/30 Read block 470 len 8192
215652024-02-06T22:24:52.973Z14/30 Write block 221 len 24576 data: 18 20 20 20 17 19
215662024-02-06T22:24:52.973Z15/30 Write block 190 len 28672 data: 13 12 9 13 17 15 17
215672024-02-06T22:24:52.973Z16/30 Flush
215682024-02-06T22:24:52.976Z17/30 Write block 347 len 4096 data: 18
215692024-02-06T22:24:52.979Z18/30 Write block 20 len 32768 data: 9 10 12 11 10 11 11 13
215702024-02-06T22:24:52.980Z19/30 Read block 291 len 36864
215712024-02-06T22:24:52.980Z20/30 Flush
215722024-02-06T22:24:52.983Z21/30 Write block 84 len 4096 data: 13
215732024-02-06T22:24:52.983Z22/30 Read block 259 len 36864
215742024-02-06T22:24:52.983Z23/30 Read block 410 len 32768
215752024-02-06T22:24:52.986Z24/30 Write block 293 len 24576 data: 14 13 16 15 11 11
215762024-02-06T22:24:52.986Z25/30 Write block 138 len 20480 data: 14 12 10 11 14
215772024-02-06T22:24:52.986Z26/30 Write block 83 len 8192 data: 14 14
215782024-02-06T22:24:52.986Z27/30 Read block 326 len 16384
215792024-02-06T22:24:52.990Z28/30 Read block 184 len 32768
215802024-02-06T22:24:52.990Z29/30 Read block 406 len 16384
215812024-02-06T22:24:52.990Z30/30 Write block 92 len 20480 data: 7 8 9 12 12
215822024-02-06T22:24:52.990Z----------------------------------------------------------------
215832024-02-06T22:24:52.990Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
215842024-02-06T22:24:52.990ZStates: Active Active Active
215852024-02-06T22:24:52.990ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
215862024-02-06T22:24:52.990Z 1 Acked 1000 Write 10 Sent Done Done false
215872024-02-06T22:24:52.990Z 2 Acked 1001 Write 3 Sent Done Done false
215882024-02-06T22:24:52.990Z 3 Acked 1002 Read 2 Sent Done Done false
215892024-02-06T22:24:52.990Z 4 Acked 1003 Read 6 Sent Done Done false
215902024-02-06T22:24:52.990Z 5 Acked 1004 Write 5 Sent Done Done false
215912024-02-06T22:24:52.990Z 6 Acked 1005 Write 3 Sent Done Done false
215922024-02-06T22:24:52.990Z 7 Acked 1006 Read 2 Sent Done Done false
215932024-02-06T22:24:52.990Z 8 Acked 1007 Read 1 Sent Done Done false
215942024-02-06T22:24:52.990Z 9 Acked 1008 Write 4 Sent Done Done false
215952024-02-06T22:24:52.990Z 10 Acked 1009 Read 9 Sent Done Done false
215962024-02-06T22:24:52.990Z 11 Acked 1010 Read 8 Sent Done Done false
215972024-02-06T22:24:52.990Z 12 Acked 1011 Write 8 Sent Done Done false
215982024-02-06T22:24:52.990Z 13 Acked 1012 Read 2 Sent Done Done false
215992024-02-06T22:24:52.991Z 14 Acked 1013 Write 6 Sent Done Done false
216002024-02-06T22:24:52.991Z 15 Acked 1014 Write 7 Sent Done Done false
216012024-02-06T22:24:52.991Z 16 Acked 1015 Flush 0 Sent Done Done false
216022024-02-06T22:24:52.991Z 17 Acked 1016 Write 1 Sent Done Done false
216032024-02-06T22:24:52.991Z 18 Acked 1017 Write 8 Sent Done Done false
216042024-02-06T22:24:52.991Z 19 Acked 1018 Read 9 Sent Done Done false
216052024-02-06T22:24:52.991Z 20 Acked 1019 Flush 0 Sent Done Done false
216062024-02-06T22:24:52.991Z 21 Acked 1020 Write 1 Sent Done Done false
216072024-02-06T22:24:52.991Z 22 Acked 1021 Read 9 Sent Done Done false
216082024-02-06T22:24:52.991Z 23 Acked 1022 Read 8 Sent Done Done false
216092024-02-06T22:24:52.991Z 24 Acked 1023 Write 6 Sent Done Done false
216102024-02-06T22:24:52.991Z 25 Acked 1024 Write 5 Sent Done Done false
216112024-02-06T22:24:52.991Z 26 Acked 1025 Write 2 Sent Done Done false
216122024-02-06T22:24:52.991Z 27 Acked 1026 Read 4 Sent Done Done false
216132024-02-06T22:24:52.991Z 28 Acked 1027 Read 8 Sent Done Done false
216142024-02-06T22:24:52.991Z 29 Acked 1028 Read 4 Sent Done Done false
216152024-02-06T22:24:52.991Z 30 Acked 1029 Write 5 Sent Sent Sent false
216162024-02-06T22:24:52.991Z STATES DS:0 DS:1 DS:2 TOTAL
216172024-02-06T22:24:52.991Z New 0 0 0 0
216182024-02-06T22:24:52.991Z Sent 30 1 1 32
216192024-02-06T22:24:52.991Z Done 0 29 29 58
216202024-02-06T22:24:52.991Z Skipped 0 0 0 0
216212024-02-06T22:24:52.991Z Error 0 0 0 0
216222024-02-06T22:24:52.992ZLast Flush: 0 1019 1019
216232024-02-06T22:24:52.992ZDownstairs last five completed:
216242024-02-06T22:24:52.992ZUpstairs last five completed: 30 29 28 27 26
216252024-02-06T22:24:52.992Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-06T22:24:52.152629982Z","hostname":"ip-10-150-1-27.us-west-2.compute.internal","pid":1933}
216262024-02-06T22:24:52.995ZWrote out file "/tmp/test_repair_verify.data"
216272024-02-06T22:24:52.995Z
216282024-02-06T22:24:52.998ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
216292024-02-06T22:24:53.013ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
216302024-02-06T22:24:53.013Z 0 000-019 195 195 195 352 352 352 F F F
216312024-02-06T22:24:53.013Z 1 020-039 196 198 198 355 358 358 F F F <---
216322024-02-06T22:24:53.013Z 2 040-059 196 196 196 355 355 355 F F F
216332024-02-06T22:24:53.013Z 3 060-079 196 196 196 355 355 355 F F F
216342024-02-06T22:24:53.013Z 4 080-099 193 193 193 344 344 344 F T T <---
216352024-02-06T22:24:53.013Z 5 100-119 194 198 198 350 357 357 F F F <---
216362024-02-06T22:24:53.013Z 6 120-139 194 194 194 347 347 347 F T T <---
216372024-02-06T22:24:53.013Z 7 140-159 194 194 194 348 348 348 F T T <---
216382024-02-06T22:24:53.013Z 8 160-179 197 198 198 356 357 357 F F F <---
216392024-02-06T22:24:53.013Z 9 180-199 197 198 198 356 357 357 F F F <---
216402024-02-06T22:24:53.013Z 10 200-219 190 198 198 339 357 357 F F F <---
216412024-02-06T22:24:53.013Z 11 220-239 195 198 198 352 357 357 F F F <---
216422024-02-06T22:24:53.013Z 12 240-259 195 195 195 352 352 352 F F F
216432024-02-06T22:24:53.013Z 13 260-279 195 195 195 352 352 352 F F F
216442024-02-06T22:24:53.016Z 14 280-299 192 192 192 343 343 343 F T T <---
216452024-02-06T22:24:53.016Z 15 300-319 189 189 189 335 335 335 F F F
216462024-02-06T22:24:53.016Z 16 320-339 196 196 196 353 353 353 F F F
216472024-02-06T22:24:53.016Z 17 340-359 197 198 198 356 358 358 F F F <---
216482024-02-06T22:24:53.017Z 18 360-379 193 193 193 344 344 344 F F F
216492024-02-06T22:24:53.017Z 19 380-399 194 194 194 349 349 349 F F F
216502024-02-06T22:24:53.017Z 20 400-419 196 196 196 354 354 354 F F F
216512024-02-06T22:24:53.017Z 21 420-439 193 193 193 344 344 344 F F F
216522024-02-06T22:24:53.017Z 22 440-459 193 193 193 344 344 344 F F F
216532024-02-06T22:24:53.017Z 23 460-479 196 196 196 355 355 355 F F F
216542024-02-06T22:24:53.017Z 24 480-499 191 191 191 340 340 340 F F F
216552024-02-06T22:24:53.017Z 25 500-519 190 198 198 339 357 357 F F F <---
216562024-02-06T22:24:53.017Z 26 520-539 194 194 194 347 347 347 F F F
216572024-02-06T22:24:53.017Z 27 540-559 192 198 198 341 357 357 F F F <---
216582024-02-06T22:24:53.017Z 28 560-579 195 198 198 352 357 357 F F F <---
216592024-02-06T22:24:53.017Z 29 580-599 194 194 194 351 351 351 F F F
216602024-02-06T22:24:53.017ZMax gen: 198, Max flush: 358
216612024-02-06T22:24:53.017ZError: Difference in extent metadata found!
216622024-02-06T22:24:53.017ZA Difference in extent metadata is expected here
216632024-02-06T22:24:53.017ZOn loop 99
216642024-02-06T22:24:53.017Z
216652024-02-06T22:24:53.020ZVerifying data now
216662024-02-06T22:24:57.705ZNo change after verify
216672024-02-06T22:24:57.705ZLoop: 99 Downstairs dump after verify (and repair):
216682024-02-06T22:24:57.718ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
216692024-02-06T22:24:57.718Z 0 000-019 195 195 195 352 352 352 F F F
216702024-02-06T22:24:57.718Z 1 020-039 198 198 198 358 358 358 F F F
216712024-02-06T22:24:57.718Z 2 040-059 196 196 196 355 355 355 F F F
216722024-02-06T22:24:57.718Z 3 060-079 196 196 196 355 355 355 F F F
216732024-02-06T22:24:57.718Z 4 080-099 199 199 199 359 359 359 F F F
216742024-02-06T22:24:57.718Z 5 100-119 198 198 198 357 357 357 F F F
216752024-02-06T22:24:57.718Z 6 120-139 199 199 199 359 359 359 F F F
216762024-02-06T22:24:57.718Z 7 140-159 199 199 199 359 359 359 F F F
216772024-02-06T22:24:57.718Z 8 160-179 198 198 198 357 357 357 F F F
216782024-02-06T22:24:57.718Z 9 180-199 198 198 198 357 357 357 F F F
216792024-02-06T22:24:57.718Z 10 200-219 198 198 198 357 357 357 F F F
216802024-02-06T22:24:57.718Z 11 220-239 198 198 198 357 357 357 F F F
216812024-02-06T22:24:57.718Z 12 240-259 195 195 195 352 352 352 F F F
216822024-02-06T22:24:57.718Z 13 260-279 195 195 195 352 352 352 F F F
216832024-02-06T22:24:57.718Z 14 280-299 199 199 199 359 359 359 F F F
216842024-02-06T22:24:57.722Z 15 300-319 189 189 189 335 335 335 F F F
216852024-02-06T22:24:57.722Z 16 320-339 196 196 196 353 353 353 F F F
216862024-02-06T22:24:57.722Z 17 340-359 198 198 198 358 358 358 F F F
216872024-02-06T22:24:57.722Z 18 360-379 193 193 193 344 344 344 F F F
216882024-02-06T22:24:57.722Z 19 380-399 194 194 194 349 349 349 F F F
216892024-02-06T22:24:57.722Z 20 400-419 196 196 196 354 354 354 F F F
216902024-02-06T22:24:57.722Z 21 420-439 193 193 193 344 344 344 F F F
216912024-02-06T22:24:57.722Z 22 440-459 193 193 193 344 344 344 F F F
216922024-02-06T22:24:57.722Z 23 460-479 196 196 196 355 355 355 F F F
216932024-02-06T22:24:57.722Z 24 480-499 191 191 191 340 340 340 F F F
216942024-02-06T22:24:57.723Z 25 500-519 198 198 198 357 357 357 F F F
216952024-02-06T22:24:57.723Z 26 520-539 194 194 194 347 347 347 F F F
216962024-02-06T22:24:57.723Z 27 540-559 198 198 198 357 357 357 F F F
216972024-02-06T22:24:57.723Z 28 560-579 198 198 198 357 357 357 F F F
216982024-02-06T22:24:57.723Z 29 580-599 194 194 194 351 351 351 F F F
216992024-02-06T22:24:57.723ZMax gen: 199, Max flush: 359
217002024-02-06T22:24:57.723Z9:27 Test duration
217012024-02-06T22:24:57.723ZTest completed
217022024-02-06T22:24:57.729Z
217032024-02-06T22:24:57.729Zreal 9:27.548329758
217042024-02-06T22:24:57.729Zuser 1:37.786393759
217052024-02-06T22:24:57.729Zsys 17.370237851
217062024-02-06T22:24:57.729Ztrap 0.085710701
217072024-02-06T22:24:57.729Ztflt 0.037834631
217082024-02-06T22:24:57.729Zdflt 0.041126248
217092024-02-06T22:24:57.729Zkflt 0.003769586
217102024-02-06T22:24:57.729Zlock 7:25:25.464843304
217112024-02-06T22:24:57.729Zslp 45:49.787732138
217122024-02-06T22:24:57.729Zlat 5.802070919
217132024-02-06T22:24:57.729Zstop 0.045026189
217142024-02-06T22:24:57.729Zprocess exited: duration 577500 ms, exit code 0
 
217152024-02-06T22:24:57.746Zfound 4 output files
217162024-02-06T22:24:57.746Zuploading: /tmp/test_repair_ds8810.txt (22007 bytes)
217172024-02-06T22:24:58.765Zuploaded: /tmp/test_repair_ds8810.txt
217182024-02-06T22:24:58.766Zuploading: /tmp/test_repair_ds8820.txt (51969 bytes)
217192024-02-06T22:24:59.785Zuploaded: /tmp/test_repair_ds8820.txt
217202024-02-06T22:24:59.785Zuploading: /tmp/test_repair_ds8830.txt (25602 bytes)
217212024-02-06T22:25:00.800Zuploaded: /tmp/test_repair_ds8830.txt
217222024-02-06T22:25:00.800Zuploading: /tmp/test_repair_out.txt (92680 bytes)
217232024-02-06T22:25:01.818Zuploaded: /tmp/test_repair_out.txt